diff --git a/render-app/src/main/java/org/janelia/alignment/match/Matches.java b/render-app/src/main/java/org/janelia/alignment/match/Matches.java index 759e574d8..c96f7d286 100644 --- a/render-app/src/main/java/org/janelia/alignment/match/Matches.java +++ b/render-app/src/main/java/org/janelia/alignment/match/Matches.java @@ -139,6 +139,19 @@ public ArrayList createPointMatches() { return matches; } + public void applyOffsets(final double pxOffset, + final double pyOffset, + final double qxOffset, + final double qyOffset) { + // NOTE: point coordinates are stored by dimension, so p[0] holds all x values and p[1] holds all y values + for (int i = 0; i < w.length; ++i) { + p[0][i] += pxOffset; + p[1][i] += pyOffset; + q[0][i] += qxOffset; + q[1][i] += qyOffset; + } + } + // NOTE: Convenience getter methods for derived data should be excluded from the JSON model - hence the JsonIgnore annotation. // Newer Jackson versions (e.g. 2.9.6) fail to parse RealPoint with an IllegalArgumentException: // Conflicting setter definitions for property "position": net.imglib2.RealPoint#setPosition(1 params) vs net.imglib2.RealPoint#setPosition(1 params) diff --git a/render-app/src/main/java/org/janelia/alignment/match/parameters/TilePairDerivationParameters.java b/render-app/src/main/java/org/janelia/alignment/match/parameters/TilePairDerivationParameters.java index 6fa7df579..e4db5f302 100644 --- a/render-app/src/main/java/org/janelia/alignment/match/parameters/TilePairDerivationParameters.java +++ b/render-app/src/main/java/org/janelia/alignment/match/parameters/TilePairDerivationParameters.java @@ -73,6 +73,12 @@ public class TilePairDerivationParameters implements Serializable { arity = 1) public boolean excludeSameMfovNeighbors = false; + @Parameter( + names = "--excludeDifferentMfovNeighbors", + description = "Exclude neighbor tiles in the same layer (z) with a different MFOV from the source tile", + arity = 1) + public boolean excludeDifferentMfovNeighbors = false; + @Parameter( names = "--excludePairsInMatchCollection", description = "Name of match collection whose existing pairs should be excluded from the generated list (default is to include all pairs)" diff --git a/render-app/src/main/java/org/janelia/alignment/multisem/MultiSemUtilities.java b/render-app/src/main/java/org/janelia/alignment/multisem/MultiSemUtilities.java index 7bca85591..f752f525e 100644 --- a/render-app/src/main/java/org/janelia/alignment/multisem/MultiSemUtilities.java +++ b/render-app/src/main/java/org/janelia/alignment/multisem/MultiSemUtilities.java @@ -263,6 +263,36 @@ public static boolean isSimpleMFOVName(final String name) { /** Each MFOV has 91 SFOVs or tiles */ public static int NUMBER_OF_TILES_IN_MFOV = 91; + /** + * @return the specified tileId with its render order updated + * (e.g.'w60_magc0399_scan005_m0013_r42_s09' for 'w60_magc0399_scan005_m0013_r37_s09') + */ + public static String convertTileIdToUseWafers6061RenderOrder(final String tileId) { + final String sfovString = getSFOVIndexForTileId(tileId); // "09" for "w60_magc0399_scan005_m0013_r37_s09" + final int i = Integer.parseInt(sfovString.trim()) - 1; // 8 for "09" + final String renderOrderString = WAFERS_60_61_RENDER_ORDER_STRINGS[i]; // "r42_" for 8 + return tileId.substring(0, 27) + renderOrderString + sfovString; + } + + /** + * Thomas' optimal SFOV render order for wafers 60 and 61 that was copied from + * + * render_sfov_order.py + * . + */ + private static final String[] WAFERS_60_61_RENDER_ORDER_STRINGS = { + "r57_s", "r50_s", "r46_s", "r51_s", "r60_s", "r65_s", "r59_s", "r47_s", "r42_s", "r36_s", + "r43_s", "r48_s", "r54_s", "r56_s", "r69_s", "r76_s", "r72_s", "r68_s", "r55_s", "r44_s", + "r37_s", "r31_s", "r24_s", "r32_s", "r38_s", "r18_s", "r25_s", "r33_s", "r39_s", "r64_s", + "r71_s", "r83_s", "r79_s", "r75_s", "r70_s", "r63_s", "r52_s", "r34_s", "r26_s", "r19_s", + "r12_s", "r07_s", "r20_s", "r27_s", "r13_s", "r08_s", "r14_s", "r21_s", "r28_s", "r35_s", + "r45_s", "r67_s", "r78_s", "r86_s", "r88_s", "r85_s", "r82_s", "r77_s", "r66_s", "r58_s", + "r49_s", "r29_s", "r22_s", "r15_s", "r09_s", "r03_s", "r01_s", "r04_s", "r16_s", "r10_s", + "r05_s", "r02_s", "r06_s", "r11_s", "r17_s", "r23_s", "r30_s", "r40_s", "r62_s", "r74_s", + "r81_s", "r90_s", "r91_s", "r89_s", "r87_s", "r84_s", "r80_s", "r73_s", "r61_s", "r53_s", + "r41_s" + }; + private static final Pattern SIMPLE_MFOV_NAME_PATTERN = Pattern.compile("^m(\\d{4})$"); private static final Logger LOG = LoggerFactory.getLogger(MultiSemUtilities.class); diff --git a/render-app/src/main/java/org/janelia/alignment/spec/ResolvedTileSpecsWithMatchPairs.java b/render-app/src/main/java/org/janelia/alignment/spec/ResolvedTileSpecsWithMatchPairs.java index 3fe215846..49e95cd34 100644 --- a/render-app/src/main/java/org/janelia/alignment/spec/ResolvedTileSpecsWithMatchPairs.java +++ b/render-app/src/main/java/org/janelia/alignment/spec/ResolvedTileSpecsWithMatchPairs.java @@ -159,6 +159,63 @@ public void normalize(final Integer maxZDistance, this.matchPairs = normalizedMatchPairs; } + /** + * Remove any match pairs where the pTile and/or the qTile do not exist in this collection's resolvedTileSpecs. + */ + public void removeMatchPairsThatReferenceTilesOutsideThisCollection() { + + final Set updatedMatchPairs = new HashSet<>(matchPairs.size()); + + for (final CanvasMatches pair : matchPairs) { + if (resolvedTileSpecs.hasTileSpec(pair.getpId()) && + resolvedTileSpecs.hasTileSpec(pair.getqId())) { + updatedMatchPairs.add(pair); + } + } + + // pairs from web service are not sorted, so sort here to make usage loops more intuitive + this.matchPairs = updatedMatchPairs.stream().sorted().collect(Collectors.toList()); + } + + /** + * Change all match points to world coordinates using tile min x and y as offset. + * + * @throws IllegalStateException + * if any match pair identifies a tile spec that is missing from this collection. + */ + public void changeMatchesToWorldCoordinates() + throws IllegalStateException { + + for (final CanvasMatches pair : matchPairs) { + + final TileSpec pTileSpec = resolvedTileSpecs.getTileSpec(pair.getpId()); + final TileSpec qTileSpec = resolvedTileSpecs.getTileSpec(pair.getqId()); + + if ((pTileSpec != null) && (qTileSpec != null)) { + + pair.getMatches().applyOffsets(pTileSpec.getMinX(), + pTileSpec.getMinY(), + qTileSpec.getMinX(), + qTileSpec.getMinY()); + + } else { + + final StringBuilder sb = new StringBuilder(); + if (qTileSpec == null) { + if (pTileSpec == null) { + sb.append("s ").append(pair.getpId()).append(" and "); + } + sb.append(" ").append(pair.getqId()); + } else { + sb.append(" ").append(pair.getpId()); + } + throw new IllegalStateException("missing tile" + sb + " for match pair " + pair.toKeyString()); + + } + } + + } + public static ResolvedTileSpecsWithMatchPairs fromJson(final Reader json) { return JSON_HELPER.fromJson(json); } diff --git a/render-app/src/main/java/org/janelia/alignment/spec/TileBoundsRTree.java b/render-app/src/main/java/org/janelia/alignment/spec/TileBoundsRTree.java index 34be18ed6..1b5f8b6ef 100644 --- a/render-app/src/main/java/org/janelia/alignment/spec/TileBoundsRTree.java +++ b/render-app/src/main/java/org/janelia/alignment/spec/TileBoundsRTree.java @@ -219,7 +219,8 @@ public Set getCircleNeighbors(final List source final boolean excludeCornerNeighbors, final boolean excludeSameLayerNeighbors, final boolean excludeSameSectionNeighbors, - final boolean excludeSameMfovNeighbors) { + final boolean excludeSameMfovNeighbors, + final boolean excludeDifferentMfovNeighbors) { String firstTileId = null; if (! sourceTileBoundsList.isEmpty()) { @@ -258,7 +259,8 @@ public Set getCircleNeighbors(final List source neighborTileIdPairs.addAll( getDistinctPairs(tileBounds, searchResults, - excludeCornerNeighbors, excludeSameSectionNeighbors, excludeSameMfovNeighbors, + excludeCornerNeighbors, excludeSameSectionNeighbors, + excludeSameMfovNeighbors, excludeDifferentMfovNeighbors, true)); } @@ -266,7 +268,8 @@ public Set getCircleNeighbors(final List source searchResults = neighborTree.findTilesInCircle(circle); neighborTileIdPairs.addAll( getDistinctPairs(tileBounds, searchResults, - excludeCornerNeighbors, excludeSameSectionNeighbors, excludeSameMfovNeighbors, + excludeCornerNeighbors, excludeSameSectionNeighbors, + excludeSameMfovNeighbors, excludeDifferentMfovNeighbors, false)); } } @@ -310,10 +313,11 @@ public static Set getDistinctPairs(final TileBounds fromTil final boolean excludeCornerNeighbors, final boolean excludeSameSectionNeighbors, final boolean excludeSameMfovNeighbors, + final boolean excludeDifferentMfovNeighbors, final boolean includeRelativePosition) { final Set pairs = new HashSet<>(toTiles.size() * 2); final String pTileId = fromTile.getTileId(); - final String pMfov = excludeSameMfovNeighbors ? MultiSemUtilities.getMagcMfovForTileId(pTileId) : null; + final String pMfov = (excludeSameMfovNeighbors || excludeDifferentMfovNeighbors) ? MultiSemUtilities.getMagcMfovForTileId(pTileId) : null; final double fromMinX = fromTile.getMinX(); final double fromMaxX = fromTile.getMaxX(); @@ -332,8 +336,11 @@ public static Set getDistinctPairs(final TileBounds fromTil isNeighborCenterInRange(fromMinX, fromMaxX, toTile.getMinX(), toTile.getMaxX()) || isNeighborCenterInRange(fromMinY, fromMaxY, toTile.getMinY(), toTile.getMaxY())) { + // note: only execute multi-SEM string extraction and comparison code if flag indicates it is needed if (excludeSameMfovNeighbors && (pMfov.equals(MultiSemUtilities.getMagcMfovForTileId(qTileId)))) { continue; + } else if (excludeDifferentMfovNeighbors && (! pMfov.equals(MultiSemUtilities.getMagcMfovForTileId(qTileId)))) { + continue; } final OrderedCanvasIdPair pair; diff --git a/render-app/src/main/java/org/janelia/alignment/spec/stack/PipelineStackIdNamingGroups.java b/render-app/src/main/java/org/janelia/alignment/spec/stack/PipelineStackIdNamingGroups.java index fa63eda48..27f128650 100644 --- a/render-app/src/main/java/org/janelia/alignment/spec/stack/PipelineStackIdNamingGroups.java +++ b/render-app/src/main/java/org/janelia/alignment/spec/stack/PipelineStackIdNamingGroups.java @@ -13,19 +13,22 @@ public class PipelineStackIdNamingGroups private final StackIdNamingGroup raw; private final StackIdNamingGroup aligned; private final StackIdNamingGroup intensityCorrected; + private final StackIdNamingGroup other; /** No-arg constructor required for JSON deserialization. */ @SuppressWarnings("unused") private PipelineStackIdNamingGroups() { - this(null, null, null); + this(null, null, null, null); } public PipelineStackIdNamingGroups(final StackIdNamingGroup raw, final StackIdNamingGroup aligned, - final StackIdNamingGroup intensityCorrected) { + final StackIdNamingGroup intensityCorrected, + final StackIdNamingGroup other) { this.raw = raw; this.aligned = aligned; this.intensityCorrected = intensityCorrected; + this.other = other; } public StackIdNamingGroup getRaw() { @@ -39,4 +42,9 @@ public StackIdNamingGroup getAligned() { public StackIdNamingGroup getIntensityCorrected() { return intensityCorrected; } + + public StackIdNamingGroup getOther() { + return other; + } + } diff --git a/render-app/src/test/java/org/janelia/alignment/spec/TileBoundsRTreeTest.java b/render-app/src/test/java/org/janelia/alignment/spec/TileBoundsRTreeTest.java index 9db430fcf..e3316f051 100644 --- a/render-app/src/test/java/org/janelia/alignment/spec/TileBoundsRTreeTest.java +++ b/render-app/src/test/java/org/janelia/alignment/spec/TileBoundsRTreeTest.java @@ -117,6 +117,7 @@ public void testGetCircleNeighbors() { false, false, false, + false, false); final Set expectedPairs = new TreeSet<>(); @@ -138,6 +139,7 @@ public void testGetCircleNeighbors() { false, false, false, + false, false); expectedPairs.clear(); @@ -168,6 +170,7 @@ public void testGetCircleNeighborsWithFullyOverlappingTiles() { false, false, false, + false, false); final Set expectedPairs = new TreeSet<>(); @@ -197,6 +200,7 @@ public void testGetCanvasIdPairs() { false, false, false, + false, false); int expectedNumberOfCombinations = tileBoundsList.size() - 1; // all tiles except the center Assert.assertEquals("incorrect number of combinations (with corner neighbors) in " + pairs, @@ -208,6 +212,7 @@ public void testGetCanvasIdPairs() { true, false, false, + false, true); Assert.assertEquals("incorrect number of combinations (without corner neighbors) in " + pairs, expectedNumberOfCombinations, pairs.size()); @@ -236,6 +241,7 @@ public void testGetDifferentMFovCanvasIdPairs() { false, false, true, + false, false); Assert.assertEquals("incorrect number of combinations in " + pairs, 2, pairs.size()); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/DebugTransformedCornersClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/DebugTransformedCornersClient.java index fdade1bfe..e576e9842 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/DebugTransformedCornersClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/DebugTransformedCornersClient.java @@ -263,6 +263,7 @@ private List debugPairsForStack(final String stackName) false, false, false, + false, false); neighborPairs.addAll(currentNeighborPairs); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/ExampleMatchVisualizationClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/ExampleMatchVisualizationClient.java index 3f0138b2f..8e556bfe5 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/ExampleMatchVisualizationClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/ExampleMatchVisualizationClient.java @@ -209,6 +209,7 @@ void printConnections() true, false, false, + false, false) .stream() .map(pair -> { @@ -246,6 +247,7 @@ void printConnections() true, true, false, + false, false) .stream().sorted().collect(Collectors.toList()); diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/RestartSolverClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/RestartSolverClient.java index b2db6a8d8..645ae4ed9 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/RestartSolverClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/RestartSolverClient.java @@ -333,6 +333,7 @@ private Set getSameLayerPairs(final Map getCrossLayerPairs(final Map sorte tpdp.excludeCornerNeighbors, tpdp.excludeSameLayerNeighbors, tpdp.excludeSameSectionNeighbors, - tpdp.excludeSameMfovNeighbors); + tpdp.excludeSameMfovNeighbors, + tpdp.excludeDifferentMfovNeighbors); if (tpdp.excludeSameLayerPairsWithPosition != null) { final MontageRelativePosition excludedPosition = tpdp.excludeSameLayerPairsWithPosition; diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/ThomasCalibrationIntensityCorrectionClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/BeamCorrectionClient.java similarity index 67% rename from render-ws-java-client/src/main/java/org/janelia/render/client/multisem/ThomasCalibrationIntensityCorrectionClient.java rename to render-ws-java-client/src/main/java/org/janelia/render/client/multisem/BeamCorrectionClient.java index b1231b0b2..8642fdb53 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/ThomasCalibrationIntensityCorrectionClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/BeamCorrectionClient.java @@ -2,14 +2,11 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.ParametersDelegate; +import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import com.google.gson.JsonParser; import com.google.gson.JsonPrimitive; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -17,13 +14,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import net.imglib2.IterableInterval; -import net.imglib2.RandomAccess; -import net.imglib2.RandomAccessibleInterval; -import net.imglib2.img.basictypeaccess.AccessFlags; -import net.imglib2.type.numeric.RealType; -import net.imglib2.type.numeric.real.FloatType; - import org.janelia.alignment.filter.FilterSpec; import org.janelia.alignment.filter.LinearIntensityMap8BitFilter; import org.janelia.alignment.spec.ResolvedTileSpecCollection; @@ -31,25 +21,36 @@ import org.janelia.alignment.spec.stack.StackMetaData; import org.janelia.render.client.ClientRunner; import org.janelia.render.client.RenderDataClient; +import org.janelia.render.client.parameter.BeamCorrectionParameters; import org.janelia.render.client.parameter.CommandLineParameters; import org.janelia.render.client.parameter.RenderWebServiceParameters; -import org.janelia.render.client.parameter.ZRangeParameters; import org.janelia.saalfeldlab.n5.DataBlock; import org.janelia.saalfeldlab.n5.DataType; import org.janelia.saalfeldlab.n5.N5Reader; +import org.janelia.saalfeldlab.n5.N5URI; import org.janelia.saalfeldlab.n5.imglib2.N5Utils; import org.janelia.saalfeldlab.n5.universe.N5Factory; +import org.janelia.saalfeldlab.n5.universe.N5Factory.StorageFormat; import org.janelia.saalfeldlab.n5.zarr.ZarrDatasetAttributes; import org.janelia.saalfeldlab.n5.zarr.ZarrKeyValueReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import net.imglib2.IterableInterval; +import net.imglib2.RandomAccess; +import net.imglib2.RandomAccessibleInterval; +import net.imglib2.img.basictypeaccess.AccessFlags; +import net.imglib2.type.numeric.RealType; +import net.imglib2.type.numeric.real.FloatType; + /** * Java client that applies the pre-computed degree-0 (spatially flat) beam-homogenization correction * for a Multi-SEM stack. *

* The correction parameters are read from a multi-SEM acquisition zarr container (the "xlog", - * e.g. xlog_wafer_61.zarr). The {@code beam_homogenization} array has dimensions + * e.g. xlog_wafer_61.zarr) that is opened with {@link N5Factory} and can therefore be either a local + * file system path or a cloud URI (e.g. gs://janelia-spark-test/xlog_data/wafer_61.zarr). + * The {@code beam_homogenization} array has dimensions * {@code [scan, slab, sfov, homogenization_parameter]}; the parameter dimension (length 26) is laid out * (per the data author's code, JaneliaSciComp/EM_recon_pipeline PR #186) as: *

    @@ -75,78 +76,18 @@ * * @author Michael Innerberger */ -public class ThomasCalibrationIntensityCorrectionClient { +public class BeamCorrectionClient { public static class Parameters extends CommandLineParameters { @ParametersDelegate public RenderWebServiceParameters renderWeb = new RenderWebServiceParameters(); - @ParametersDelegate - public ZRangeParameters layerRange = new ZRangeParameters(); - @Parameter(names = "--stack", description = "Source stack to correct", required = true) public String stack; - @Parameter(names = "--targetStack", description = "Name of the derived stack to store the corrected " + - "tile specs (defaults to _ic)") - public String targetStack; - - @Parameter(names = "--zarrPath", description = "Path to the multi-SEM acquisition zarr (xlog) container " + - "holding the intensity correction parameters and the scan/slab/sfov coordinate arrays " + - "(e.g. /path/to/xlog_wafer_61.zarr)", required = true) - public String zarrPath; - - @Parameter(names = "--homogenizationDataset", description = "Name of the 4D correction-parameter array " + - "within the zarr container") - public String homogenizationDataset = "beam_homogenization"; - - @Parameter(names = "--scanDataset", description = "Name of the 1D scan coordinate array") - public String scanDataset = "scan"; - - @Parameter(names = "--slabDataset", description = "Name of the 1D slab coordinate array " + - "(labeled by the magc number of each tile)") - public String slabDataset = "slab"; - - @Parameter(names = "--sfovDataset", description = "Name of the 1D sfov coordinate array") - public String sfovDataset = "sfov"; - - @Parameter(names = "--serialDataset", description = "Name of the 1D serial-section coordinate array " + - "(only used to cross-check / log the slab selection)") - public String serialDataset = "id_serial"; - - @Parameter(names = "--gainIndex", description = "Index of the gain parameter within the " + - "homogenization_parameter dimension") - public int gainIndex = 21; - - @Parameter(names = "--deg0Index", description = "Index of the degree-0 flat-level parameter within the " + - "homogenization_parameter dimension") - public int deg0Index = 22; - - @Parameter(names = "--referenceLevel", description = "Reference intensity level to map to " + - "(defaults to the 'b_ref' attribute of the homogenization array)") - public Double referenceLevel; - - @Parameter(names = "--inverted", description = "Indicates that the source images are intensity-inverted " + - "(in' = 255 - in) relative to the data the homogenization parameters were computed for. When set, " + - "the correction is applied in the original (non-inverted) domain and the result is re-inverted, so " + - "the slope stays gain but the offset becomes 255*(1 - gain) - (referenceLevel - gain*degree0).", arity = 0) - public boolean inverted = false; - - @Parameter(names = "--sfovLabelOffset", description = "Offset added to the sFOV number parsed from the " + - "tile id to obtain the xlog sfov coordinate label. Render tile ids are 1-based (s01..s91) while " + - "the xlog sfov coordinate is 0-based (0..90), so the default is -1.") - public int sfovLabelOffset = -1; - - @Parameter(names = "--z", description = "Explicit z values for sections to be processed", variableArity = true) - public List zValues; - - @Parameter(names = "--completeStack", description = "Complete the target stack after processing", arity = 0) - public boolean completeStack = false; - - public String getTargetStack() { - return (targetStack == null || targetStack.isEmpty()) ? stack + "_ic" : targetStack; - } + @ParametersDelegate + public BeamCorrectionParameters beam = new BeamCorrectionParameters(); } public static void main(final String[] args) { @@ -157,8 +98,14 @@ public void runClient(final String[] args) throws Exception { parameters.parse(args); LOG.info("runClient: entry, parameters={}", parameters); - final ThomasCalibrationIntensityCorrectionClient client = new ThomasCalibrationIntensityCorrectionClient(parameters); - client.correctStack(); + parameters.beam.validate(); + + final BeamCorrectionClient client = new BeamCorrectionClient(); + final RenderDataClient dataClient = parameters.renderWeb.getDataClient(); + client.correctStack(dataClient, + parameters.stack, + parameters.beam, + true); } }; clientRunner.run(); @@ -171,69 +118,68 @@ public void runClient(final String[] args) throws Exception { private static final double IMAGE_MAX = 255.0; private static final String B_REF_KEY = "b_ref"; + private static final String ZMETADATA_FILE = ".zmetadata"; - private final Parameters params; - - public ThomasCalibrationIntensityCorrectionClient(final Parameters params) { - this.params = params; + public BeamCorrectionClient() { } - public void correctStack() throws IOException { + public void correctStack(final RenderDataClient dataClient, + final String stack, + final BeamCorrectionParameters beam, + final boolean completeStack) throws IOException { + + final String targetStack = beam.getTargetStack(stack); - final RenderDataClient dataClient = params.renderWeb.getDataClient(); - final String targetStack = params.getTargetStack(); + try (final N5Reader reader = new N5Factory().openReader(StorageFormat.ZARR, beam.zarrPath)) { + LOG.info("correctStack: opened {} using {}", beam.zarrPath, reader.getClass().getSimpleName()); - try (final N5Reader reader = new N5Factory().openReader(params.zarrPath)) { - LOG.info("correctStack: opened {} using {}", params.zarrPath, reader.getClass().getSimpleName()); + final ZarrKeyValueReader zarrReader = asZarrReader(reader, beam.zarrPath); - final double referenceLevel = resolveReferenceLevel(); + final double referenceLevel = resolveReferenceLevel(zarrReader, beam); // coordinate-value (label) -> array-position lookups - final Map scanLabelToPosition = readCoordinateIndex(reader, params.scanDataset); - final Map slabLabelToPosition = readCoordinateIndex(reader, params.slabDataset); - final Map sfovLabelToPosition = readCoordinateIndex(reader, params.sfovDataset); - final int[] serialBySlabPosition = readOptionalIntArray(reader, params.serialDataset); + final Map scanLabelToPosition = readCoordinateIndex(zarrReader, beam.scanDataset); + final Map slabLabelToPosition = readCoordinateIndex(zarrReader, beam.slabDataset); + final Map sfovLabelToPosition = readCoordinateIndex(zarrReader, beam.sfovDataset); + final int[] serialBySlabPosition = readOptionalIntArray(zarrReader, beam.serialDataset); LOG.info("correctStack: reference level (b_ref) is {}, source data inverted is {}; coordinate arrays loaded - {} scans, {} slabs, {} sfovs", - referenceLevel, params.inverted, scanLabelToPosition.size(), slabLabelToPosition.size(), sfovLabelToPosition.size()); + referenceLevel, beam.inverted, scanLabelToPosition.size(), slabLabelToPosition.size(), sfovLabelToPosition.size()); // the 4D correction array; axes are matched to coordinate sizes so the code is robust to axis order - final RandomAccessibleInterval> homogenization = openHomogenizationArray(reader, params.homogenizationDataset); + final RandomAccessibleInterval> homogenization = openHomogenizationArray(reader, beam.homogenizationDataset); final int scanAxis = findAxisForSize(homogenization, scanLabelToPosition.size(), "scan"); final int slabAxis = findAxisForSize(homogenization, slabLabelToPosition.size(), "slab"); final int sfovAxis = findAxisForSize(homogenization, sfovLabelToPosition.size(), "sfov"); final int parameterAxis = remainingAxis(homogenization, scanAxis, slabAxis, sfovAxis); final long parameterCount = homogenization.dimension(parameterAxis); - if (params.gainIndex < 0 || params.gainIndex >= parameterCount - || params.deg0Index < 0 || params.deg0Index >= parameterCount) { - throw new IllegalArgumentException("gainIndex " + params.gainIndex + " and deg0Index " + params.deg0Index + + if (beam.gainIndex < 0 || beam.gainIndex >= parameterCount + || beam.deg0Index < 0 || beam.deg0Index >= parameterCount) { + throw new IllegalArgumentException("gainIndex " + beam.gainIndex + " and deg0Index " + beam.deg0Index + " must both be within the parameter dimension of size " + parameterCount); } LOG.info("correctStack: '{}' axis mapping is scan={}, slab={}, sfov={}, parameter={} (size {}); using gainIndex={}, deg0Index={}", - params.homogenizationDataset, scanAxis, slabAxis, sfovAxis, parameterAxis, parameterCount, - params.gainIndex, params.deg0Index); + beam.homogenizationDataset, scanAxis, slabAxis, sfovAxis, parameterAxis, parameterCount, + beam.gainIndex, beam.deg0Index); final RandomAccess> access = homogenization.randomAccess(); final long[] position = new long[homogenization.numDimensions()]; - final List zValues = dataClient.getStackZValues(params.stack, - params.layerRange.minZ, - params.layerRange.maxZ, - params.zValues); + final List zValues = dataClient.getStackZValues(stack); if (zValues.isEmpty()) { - throw new IllegalArgumentException("source stack " + params.stack + " does not contain any matching z values"); + throw new IllegalArgumentException("source stack " + stack + " does not contain any matching z values"); } - final StackMetaData sourceStackMetaData = dataClient.getStackMetaData(params.stack); + final StackMetaData sourceStackMetaData = dataClient.getStackMetaData(stack); dataClient.setupDerivedStack(sourceStackMetaData, targetStack); int correctedCount = 0; int skippedCount = 0; for (final Double z : zValues) { - final ResolvedTileSpecCollection resolvedTiles = dataClient.getResolvedTiles(params.stack, z); + final ResolvedTileSpecCollection resolvedTiles = dataClient.getResolvedTiles(stack, z); for (final TileSpec tileSpec : resolvedTiles.getTileSpecs()) { final String tileId = tileSpec.getTileId(); @@ -241,7 +187,7 @@ public void correctStack() throws IOException { final int magc = parseValue(MAGC_PATTERN, tileId, "magc"); final int scan = parseValue(SCAN_PATTERN, tileId, "scan"); final int sfov = parseValue(SFOV_PATTERN, tileId, "sfov"); - final int sfovLabel = sfov + params.sfovLabelOffset; + final int sfovLabel = sfov + beam.sfovLabelOffset; final Integer scanPosition = scanLabelToPosition.get(scan); final Integer slabPosition = slabLabelToPosition.get(magc); @@ -258,11 +204,11 @@ public void correctStack() throws IOException { position[slabAxis] = slabPosition; position[sfovAxis] = sfovPosition; - position[parameterAxis] = params.gainIndex; + position[parameterAxis] = beam.gainIndex; access.setPosition(position); final double gain = access.get().getRealDouble(); - position[parameterAxis] = params.deg0Index; + position[parameterAxis] = beam.deg0Index; access.setPosition(position); final double degree0 = access.get().getRealDouble(); @@ -281,7 +227,7 @@ public void correctStack() throws IOException { tileId, scanPosition, slabPosition, serialInfo, sfovPosition, gain, degree0); } - tileSpec.setFilterSpec(buildHomogenizationFilterSpec(gain, degree0, referenceLevel, params.inverted)); + tileSpec.setFilterSpec(buildHomogenizationFilterSpec(gain, degree0, referenceLevel, beam.inverted)); tileSpec.convertSingleChannelSpecToLegacyForm(); correctedCount++; } @@ -290,12 +236,12 @@ public void correctStack() throws IOException { LOG.info("correctStack: saved z {} to {}", z, targetStack); } - if (params.completeStack) { + if (completeStack) { dataClient.setStackState(targetStack, StackMetaData.StackState.COMPLETE); } LOG.info("correctStack: exit, applied degree-0 homogenization to {} tiles and skipped {} tiles across {} layers of {} (target stack {})", - correctedCount, skippedCount, zValues.size(), params.stack, targetStack); + correctedCount, skippedCount, zValues.size(), stack, targetStack); } } @@ -324,54 +270,82 @@ private static FilterSpec buildHomogenizationFilterSpec(final double gain, return FilterSpec.forFilter(filter); } - private double resolveReferenceLevel() { - if (params.referenceLevel != null) { - return params.referenceLevel; + private double resolveReferenceLevel(final ZarrKeyValueReader reader, + final BeamCorrectionParameters beam) { + if (beam.referenceLevel != null) { + return beam.referenceLevel; } // NOTE: n5-zarr's reader.getAttribute(...) throws an NPE on blosc-compressed arrays because it - // serializes the compressor, so read the b_ref value directly from the on-disk JSON metadata. - Double bRef = readDoubleFromJsonFile(Paths.get(params.zarrPath, params.homogenizationDataset, ".zattrs"), - B_REF_KEY); + // serializes the compressor, so read the b_ref value directly from the JSON metadata. + Double bRef = readDouble(readJsonResource(reader, beam.homogenizationDataset, ZarrKeyValueReader.ZATTRS_FILE), + B_REF_KEY); if (bRef == null) { - bRef = readBRefFromConsolidatedMetadata(); + bRef = readDouble(readConsolidatedJson(reader, beam.homogenizationDataset, ZarrKeyValueReader.ZATTRS_FILE), + B_REF_KEY); } if (bRef == null) { throw new IllegalArgumentException("could not read '" + B_REF_KEY + "' from the " + - params.homogenizationDataset + " metadata under " + params.zarrPath + + beam.homogenizationDataset + " metadata under " + beam.zarrPath + "; specify --referenceLevel explicitly"); } LOG.info("resolveReferenceLevel: using reference level {}={} from zarr metadata", B_REF_KEY, bRef); return bRef; } - private Double readBRefFromConsolidatedMetadata() { + /** + * Reads a JSON metadata resource (e.g. {@code .zarray} or {@code .zattrs}) for a dataset, + * falling back to the consolidated {@code .zmetadata} at the container root. + *

    + * The resource is read with the container's {@code KeyValueAccess} instead of {@link java.nio.file.Files} + * so that metadata can be read from a local file system path or from a cloud URI + * (e.g. {@code gs://bucket/xlog_wafer_61.zarr}). + * + * @return the parsed resource or null if it is not present in the container. + */ + private static JsonObject readDatasetJson(final ZarrKeyValueReader reader, + final String dataset, + final String resourceName) { + + final JsonObject json = readJsonResource(reader, dataset, resourceName); + return json == null ? readConsolidatedJson(reader, dataset, resourceName) : json; + } + + /** Reads a dataset's JSON metadata resource from the consolidated {@code .zmetadata} at the container root. */ + private static JsonObject readConsolidatedJson(final ZarrKeyValueReader reader, + final String dataset, + final String resourceName) { + final JsonObject consolidated = readJsonResource(reader, "", ZMETADATA_FILE); + final JsonObject metadata = consolidated == null ? null : consolidated.getAsJsonObject("metadata"); + final JsonObject datasetJson = metadata == null ? null : metadata.getAsJsonObject(dataset + "/" + resourceName); + return datasetJson == null ? null : datasetJson.deepCopy(); + } + + private static Double readDouble(final JsonObject json, + final String key) { + return ((json != null) && json.has(key)) ? json.get(key).getAsDouble() : null; + } + + /** Reads one JSON resource from the container, or returns null if it is missing or cannot be parsed. */ + private static JsonObject readJsonResource(final ZarrKeyValueReader reader, + final String parentPath, + final String resourceName) { try { - final Path zMetadata = Paths.get(params.zarrPath, ".zmetadata"); - if (!Files.isRegularFile(zMetadata)) { - return null; - } - final JsonObject root = JsonParser.parseString(Files.readString(zMetadata)).getAsJsonObject(); - final JsonObject metadata = root.getAsJsonObject("metadata"); - if (metadata == null) { - return null; - } - final JsonObject attrs = metadata.getAsJsonObject(params.homogenizationDataset + "/.zattrs"); - return (attrs != null && attrs.has(B_REF_KEY)) ? attrs.get(B_REF_KEY).getAsDouble() : null; + final JsonElement element = reader.getAttributesFromContainer(N5URI.normalizeGroupPath(parentPath), + resourceName); + return ((element != null) && element.isJsonObject()) ? element.getAsJsonObject() : null; } catch (final Exception e) { + LOG.warn("readJsonResource: failed to read {} for '{}' ({})", resourceName, parentPath, e.getMessage()); return null; } } - private static Double readDoubleFromJsonFile(final Path path, final String key) { - try { - if (!Files.isRegularFile(path)) { - return null; - } - final JsonObject obj = JsonParser.parseString(Files.readString(path)).getAsJsonObject(); - return obj.has(key) ? obj.get(key).getAsDouble() : null; - } catch (final Exception e) { - return null; + private static ZarrKeyValueReader asZarrReader(final N5Reader reader, + final String zarrPath) { + if (!(reader instanceof ZarrKeyValueReader)) { + throw new IllegalArgumentException("expected a zarr reader for " + zarrPath + + " but got " + reader.getClass().getName()); } + return (ZarrKeyValueReader) reader; } private static int parseValue(final Pattern pattern, final String tileId, final String label) { @@ -383,7 +357,8 @@ private static int parseValue(final Pattern pattern, final String tileId, final } /** Reads a 1D coordinate array and returns a map from each stored label value to its array position. */ - private Map readCoordinateIndex(final N5Reader reader, final String dataset) { + private static Map readCoordinateIndex(final ZarrKeyValueReader reader, + final String dataset) { final double[] values = readCoordinateValues(reader, dataset); if (values == null) { throw new IllegalArgumentException("coordinate array '" + dataset + "' not found in the zarr container; " + @@ -397,7 +372,8 @@ private Map readCoordinateIndex(final N5Reader reader, final S } /** Reads a 1D array into an int array indexed by position, or returns null if the dataset is absent. */ - private int[] readOptionalIntArray(final N5Reader reader, final String dataset) { + private static int[] readOptionalIntArray(final ZarrKeyValueReader reader, + final String dataset) { final double[] values; try { values = readCoordinateValues(reader, dataset); @@ -425,18 +401,16 @@ private int[] readOptionalIntArray(final N5Reader reader, final String dataset) * that are physically present), the resulting {@link ZarrDatasetAttributes} is built via the reader, and the * chunks are read with {@link N5Reader#readBlock} (which honors the zarr little-endian byte order). */ - private double[] readCoordinateValues(final N5Reader reader, final String dataset) { - final JsonObject zArray = readZArrayJson(dataset); + private static double[] readCoordinateValues(final ZarrKeyValueReader reader, + final String dataset) { + final JsonObject zArray = readDatasetJson(reader, dataset, ZarrKeyValueReader.ZARRAY_FILE); if (zArray == null) { return null; } if (!zArray.has("fill_value") || zArray.get("fill_value").isJsonNull()) { zArray.add("fill_value", new JsonPrimitive("0")); } - if (!(reader instanceof ZarrKeyValueReader)) { - throw new IllegalArgumentException("expected a zarr reader but got " + reader.getClass().getName()); - } - final ZarrDatasetAttributes attributes = ((ZarrKeyValueReader) reader).createDatasetAttributes(zArray); + final ZarrDatasetAttributes attributes = reader.createDatasetAttributes(zArray); if (attributes == null) { throw new IllegalArgumentException("could not parse .zarray for coordinate dataset " + dataset); } @@ -451,7 +425,7 @@ private double[] readCoordinateValues(final N5Reader reader, final String datase final double[] values = new double[length]; final int numChunks = (int) Math.ceil((double) length / chunkSize); for (int chunk = 0; chunk < numChunks; chunk++) { - final DataBlock block = reader.readBlock(dataset, attributes, (long) chunk); + final DataBlock block = reader.readBlock(dataset, attributes, chunk); if (block == null) { throw new IllegalArgumentException("missing chunk " + chunk + " of coordinate array " + dataset); } @@ -460,34 +434,6 @@ private double[] readCoordinateValues(final N5Reader reader, final String datase return values; } - /** Reads the raw .zarray JSON for a dataset from the per-array file, falling back to consolidated .zmetadata. */ - private JsonObject readZArrayJson(final String dataset) { - final Path perArray = Paths.get(params.zarrPath, dataset, ".zarray"); - try { - if (Files.isRegularFile(perArray)) { - return JsonParser.parseString(Files.readString(perArray)).getAsJsonObject(); - } - } catch (final Exception e) { - LOG.warn("readZArrayJson: failed to read {} ({})", perArray, e.getMessage()); - } - try { - final Path zMetadata = Paths.get(params.zarrPath, ".zmetadata"); - if (Files.isRegularFile(zMetadata)) { - final JsonObject root = JsonParser.parseString(Files.readString(zMetadata)).getAsJsonObject(); - final JsonObject metadata = root.getAsJsonObject("metadata"); - if (metadata != null) { - final JsonObject zArray = metadata.getAsJsonObject(dataset + "/.zarray"); - if (zArray != null) { - return zArray.deepCopy(); - } - } - } - } catch (final Exception e) { - LOG.warn("readZArrayJson: failed to read .zmetadata for {} ({})", dataset, e.getMessage()); - } - return null; - } - /** Copies a decoded data block into dst[offset...], applying unsigned promotion based on the data type. */ private static void copyBlockValues(final Object data, final DataType dataType, final double[] dst, final int offset) { if (data instanceof byte[]) { @@ -575,5 +521,5 @@ private static int remainingAxis(final RandomAccessibleInterval rai, final in throw new IllegalArgumentException("could not identify the parameter axis of the correction array"); } - private static final Logger LOG = LoggerFactory.getLogger(ThomasCalibrationIntensityCorrectionClient.class); + private static final Logger LOG = LoggerFactory.getLogger(BeamCorrectionClient.class); } diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MFOVAsTileMontageMatchPatchClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MFOVAsTileMontageMatchPatchClient.java index 300a0680f..a363e2d5b 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MFOVAsTileMontageMatchPatchClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MFOVAsTileMontageMatchPatchClient.java @@ -153,6 +153,7 @@ private static int patchUnconnectedPairsForZ(final String stack, false, false, false, + false, false); // add all tile pairs to unconnected set to start diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MFOVMontageMatchPatchClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MFOVMontageMatchPatchClient.java index 4972782bc..e173efcaf 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MFOVMontageMatchPatchClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/MFOVMontageMatchPatchClient.java @@ -50,13 +50,6 @@ * to ensure that standard matches are given precedence. * *

  • - * Step 1: If a sameLayerDerivedMatchWeight is specified and unconnected pairs exist, - * the client loops through each unconnected pair and tries to find existing matches for that same pair in another - * MFOV in the same layer. If found, the existing matches are copied to the unconnected pair and stored - * with the sameLayerDerivedMatchWeight which is typically some reduced value like 0.15 - * to ensure that standard matches are given precedence. - *
    • - *
  • * Step 2: If a crossLayerDerivedMatchWeight is specified and unconnected pairs remain after step 1, * the client collects matches for the unconnected pair in all other layers and fits them to a * "montage patch match model". Montage patch matches are derived by applying the model to each SFOV tile's @@ -258,6 +251,7 @@ public void updatePositionPairDataForZ(final String stack, false, false, false, + false, false); // add all MFOV tile pairs to unconnected set to start diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/UnconnectedMontageMFOVClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/UnconnectedMontageMFOVClient.java index fa10312b2..d9bd115c8 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/UnconnectedMontageMFOVClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/multisem/UnconnectedMontageMFOVClient.java @@ -57,10 +57,19 @@ public static class Parameters @Parameter( names = "--startPositionMatchWeight", description = "Weight (e.g. 0.001) for matches derived from SFOV start positions. " + - "Specify to patch all unconnected pairs with positions based upon SFOV stage locations. " + - "Omit to skip start position derivation.") + "Specify a positive value to patch all unconnected pairs with positions based upon " + + "SFOV stage locations. " + + "Omit (or specify a non-positive value) to skip start position derivation.") public Double startPositionMatchWeight; + /** + * @return the start position match weight or null if derivation should be skipped + * (non-positive weights are used to exclude cross MFOV pairs from patching). + */ + public Double getStartPositionMatchWeight() { + return ((startPositionMatchWeight != null) && (startPositionMatchWeight > 0.0)) ? + startPositionMatchWeight : null; + } } /** Label for tiles in MFOVs with isolated edges. */ @@ -105,7 +114,7 @@ public void findIsolatedMFOVs() parameters.multiProject.deriveMatchCollectionNamesFromProject, renderDataClient, parameters.addIsolatedEdgeLabel, - parameters.startPositionMatchWeight); + parameters.getStartPositionMatchWeight()); } } diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/BeamCorrectionParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/BeamCorrectionParameters.java new file mode 100644 index 000000000..f7a068d6b --- /dev/null +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/BeamCorrectionParameters.java @@ -0,0 +1,125 @@ +package org.janelia.render.client.parameter; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import java.io.Serializable; + +/** + * Parameters for correcting multi-SEM image creep. + */ +@Parameters +public class BeamCorrectionParameters + implements Serializable { + + @Parameter( + names = "--zarrPath", + description = "Path or URI for the multi-SEM acquisition zarr (xlog) container " + + "holding the intensity correction parameters and the scan/slab/sfov coordinate arrays " + + "(e.g. /groups/hess/hesslab/ibeammsem/system_02/wafers/wafer_61/xlog/xlog_wafer_61.zarr or " + + "gs://janelia-spark-test/library/xlog_wafer_61.zarr)", + required = true) + public String zarrPath; + + @Parameter( + names = "--homogenizationDataset", + description = "Name of the 4D correction-parameter array within the zarr container") + public String homogenizationDataset = "beam_homogenization"; + + @Parameter( + names = "--scanDataset", + description = "Name of the 1D scan coordinate array") + public String scanDataset = "scan"; + + @Parameter( + names = "--slabDataset", + description = "Name of the 1D slab coordinate array (labeled by the magc number of each tile)") + public String slabDataset = "slab"; + + @Parameter( + names = "--sfovDataset", + description = "Name of the 1D sfov coordinate array") + public String sfovDataset = "sfov"; + + @Parameter( + names = "--serialDataset", + description = "Name of the 1D serial-section coordinate array (only used to cross-check / log the slab selection)") + public String serialDataset = "id_serial"; + + @Parameter( + names = "--gainIndex", + description = "Index of the gain parameter within the homogenization_parameter dimension") + public int gainIndex = 21; + + @Parameter( + names = "--deg0Index", + description = "Index of the degree-0 flat-level parameter within the homogenization_parameter dimension") + public int deg0Index = 22; + + @Parameter( + names = "--referenceLevel", + description = "Reference intensity level to map to (defaults to the 'b_ref' attribute of the homogenization array)") + public Double referenceLevel; + + @Parameter( + names = "--inverted", + description = "Indicates that the source images are intensity-inverted " + + "(in' = 255 - in) relative to the data the homogenization parameters were computed for. When set, " + + "the correction is applied in the original (non-inverted) domain and the result is re-inverted, so " + + "the slope stays gain but the offset becomes 255*(1 - gain) - (referenceLevel - gain*degree0).", + arity = 0) + public boolean inverted = false; + + @Parameter( + names = "--sfovLabelOffset", + description = "Offset added to the sFOV number parsed from the " + + "tile id to obtain the xlog sfov coordinate label. Render tile ids are 1-based (s01..s91) while " + + "the xlog sfov coordinate is 0-based (0..90), so the default is -1.") + public int sfovLabelOffset = -1; + + @Parameter( + names = "--targetStackSuffix", + description = "Target stack name is the the source stack name with this suffix appended") + public String targetStackSuffix = "_bc"; + + @Parameter( + names = "--overwriteSourceStack", + description = "Indicates that beam corrected tile specs should be written back to the source stack. " + + "When specified, this parameter will override the targetStackSuffix parameter.", + arity = 0) + public boolean overwriteSourceStack = false; + + public BeamCorrectionParameters() { + } + + public void validate() + throws IllegalArgumentException { + + if ((targetStackSuffix == null) || (targetStackSuffix.trim().isEmpty())) { + throw new IllegalArgumentException("--targetStackSuffix must be defined"); + } + + } + + @Override + public String toString() { + return "{zarrPath='" + zarrPath + '\'' + + ", homogenizationDataset='" + homogenizationDataset + '\'' + + ", scanDataset='" + scanDataset + '\'' + + ", slabDataset='" + slabDataset + '\'' + + ", sfovDataset='" + sfovDataset + '\'' + + ", serialDataset='" + serialDataset + '\'' + + ", gainIndex=" + gainIndex + + ", deg0Index=" + deg0Index + + ", referenceLevel=" + referenceLevel + + ", inverted=" + inverted + + ", sfovLabelOffset=" + sfovLabelOffset + + ", targetStackSuffix='" + targetStackSuffix + '\'' + + ", overwriteSourceStack=" + overwriteSourceStack + + '}'; + } + + public String getTargetStack(final String sourceStack) { + return overwriteSourceStack ? sourceStack : sourceStack + targetStackSuffix; + } +} \ No newline at end of file diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/CreepCorrectionParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/CreepCorrectionParameters.java index 6198b14c5..d4ac0939a 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/CreepCorrectionParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/CreepCorrectionParameters.java @@ -15,6 +15,13 @@ public class CreepCorrectionParameters implements Serializable { + public enum MatchCorrectionType { + OVERWRITE_SOURCE, + OVERWRITE_SOURCE_AND_RENAME_AS_TARGET, + WRITE_TO_TARGET, + SKIP + } + @Parameter( names = "--creepTargetStackSuffix", description = "Target stack name is the the source stack name with this suffix appended") @@ -31,9 +38,9 @@ public class CreepCorrectionParameters public Double maxZ; @Parameter( - names = "--skipMatchCorrection", - description = "Skip transforming match coordinates (default is to transform them)") - public boolean skipMatchCorrection = false; + names = "--matchCorrectionType", + description = "Type of match correction") + public MatchCorrectionType matchCorrectionType = MatchCorrectionType.WRITE_TO_TARGET; @Parameter( names = "--parameterCsvDir", @@ -45,6 +52,22 @@ public class CreepCorrectionParameters public CreepCorrectionParameters() { } + public boolean correctMatches() { + return matchCorrectionType != MatchCorrectionType.SKIP; + } + + public boolean overwriteMatchData() { + return matchCorrectionType == MatchCorrectionType.OVERWRITE_SOURCE; + } + + public boolean overwriteMatchDataAndRenameAsTarget() { + return matchCorrectionType == MatchCorrectionType.OVERWRITE_SOURCE_AND_RENAME_AS_TARGET; + } + + public boolean writeMatchDataToTargetCollection() { + return matchCorrectionType == MatchCorrectionType.WRITE_TO_TARGET; + } + public void validate() throws IllegalArgumentException { @@ -60,6 +83,16 @@ public void validate() } } + @Override + public String toString() { + return "{targetStackSuffix='" + targetStackSuffix + '\'' + + ", minZ=" + minZ + + ", maxZ=" + maxZ + + ", matchCorrectionType=" + matchCorrectionType + + ", parameterCsvDir='" + parameterCsvDir + '\'' + + '}'; + } + public String getTargetStack(final String sourceStack) { return sourceStack + targetStackSuffix; } diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MFOVAsTileParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MFOVAsTileParameters.java index 93a8d985a..753f3af5b 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MFOVAsTileParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MFOVAsTileParameters.java @@ -36,6 +36,7 @@ public class MFOVAsTileParameters private final String alignedMfovStackSuffix; private final String roughSfovStackSuffix; private final Double minCrossMatchPixelDistance; + private final boolean deriveSfovMatchData; public MFOVAsTileParameters() { this(null, @@ -62,7 +63,8 @@ public MFOVAsTileParameters(final Double mfovRenderScale, null, alignedMfovStackSuffix, roughSfovStackSuffix, - null); + null, + true); } public MFOVAsTileParameters(final Double mfovRenderScale, @@ -73,7 +75,8 @@ public MFOVAsTileParameters(final Double mfovRenderScale, final String renderedMfovRunTimestamp, final String alignedMfovStackSuffix, final String roughSfovStackSuffix, - final Double minCrossMatchPixelDistance) { + final Double minCrossMatchPixelDistance, + final boolean deriveSfovMatchData) { this.mfovRenderScale = mfovRenderScale; this.mfovRootDirectory = mfovRootDirectory; this.prealignedSfovStackSuffix = prealignedSfovStackSuffix; @@ -83,6 +86,7 @@ public MFOVAsTileParameters(final Double mfovRenderScale, this.alignedMfovStackSuffix = alignedMfovStackSuffix; this.roughSfovStackSuffix = roughSfovStackSuffix; this.minCrossMatchPixelDistance = minCrossMatchPixelDistance; + this.deriveSfovMatchData = deriveSfovMatchData; } public Double getMfovRenderScale() { @@ -117,6 +121,10 @@ public Double getMinCrossMatchPixelDistance() { return minCrossMatchPixelDistance; } + public boolean isDeriveSfovMatchData() { + return deriveSfovMatchData; + } + public String getDynamicMfovStackSuffixForRawSfovStack() { return prealignedSfovStackSuffix + dynamicMfovStackSuffix; } diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MFOVMontageMatchPatchParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MFOVMontageMatchPatchParameters.java index 763550342..e99755edd 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MFOVMontageMatchPatchParameters.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MFOVMontageMatchPatchParameters.java @@ -213,8 +213,18 @@ public String getTrimStackName(final String sourceStackName) { return trimMfovsWithNoConnectedTiles ? sourceStackName + "_trim" : null; } + /** + * @return the weight for matches derived from SFOV start positions + * or null if start position derivation should be skipped + * (non-positive weights are used to exclude cross MFOV pairs from patching). + */ + public Double getResinMfovStartPositionMatchWeight() { + return ((resinMfovStartPositionMatchWeight != null) && (resinMfovStartPositionMatchWeight > 0.0)) ? + resinMfovStartPositionMatchWeight : null; + } + public boolean isIsolatedMfovPatchingNeeded() { - return addIsolatedEdgeLabel || resinMfovStartPositionMatchWeight != null; + return addIsolatedEdgeLabel || (getResinMfovStartPositionMatchWeight() != null); } public static MFOVMontageMatchPatchParameters fromJson(final Reader json) { diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchCollectionRenameParameters.java b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchCollectionRenameParameters.java new file mode 100644 index 000000000..b0ca50b39 --- /dev/null +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/parameter/MatchCollectionRenameParameters.java @@ -0,0 +1,72 @@ +package org.janelia.render.client.parameter; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import java.io.Serializable; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; + +/** + * Parameters for renaming match collections. + */ +@Parameters +public class MatchCollectionRenameParameters + implements Serializable { + + @Parameter( + names = "--matchCollectionOwner", + description = "Owner of the match collections to rename") + public String matchCollectionOwner; + + @Parameter( + names = "--sourceNamePattern", + description = "Regular expression for the names of the collections to rename " + + "(e.g. '(w6.*_gc_icc)_match')") + public String sourceNamePattern; + + @Parameter( + names = "--targetNamePattern", + description = "Replacement for renamed collections that can reference capture groups " + + "from the sourceNamePattern (e.g. '$1_par_match' renames " + + "w61_s140_r00_gc_icc_match to w61_s140_r00_gc_icc_par_match)") + public String targetNamePattern; + + public MatchCollectionRenameParameters() { + } + + public void validate() + throws IllegalArgumentException { + + if ((matchCollectionOwner == null) || (matchCollectionOwner.trim().isEmpty())) { + throw new IllegalArgumentException("--matchCollectionOwner must be defined"); + } + + if ((sourceNamePattern == null) || (sourceNamePattern.trim().isEmpty())) { + throw new IllegalArgumentException("--sourceNamePattern must be defined"); + } + + if ((targetNamePattern == null) || (targetNamePattern.trim().isEmpty())) { + throw new IllegalArgumentException("--targetNamePattern must be defined"); + } + + buildSourceNamePattern(); // throws exception if the pattern is invalid + } + + public Pattern buildSourceNamePattern() + throws IllegalArgumentException { + try { + return Pattern.compile(sourceNamePattern); + } catch (final PatternSyntaxException e) { + throw new IllegalArgumentException("invalid --sourceNamePattern '" + sourceNamePattern + "'", e); + } + } + + @Override + public String toString() { + return "{owner='" + matchCollectionOwner + '\'' + + ", sourceNamePattern='" + sourceNamePattern + '\'' + + ", targetNamePattern='" + targetNamePattern + '\'' + + '}'; + } +} diff --git a/render-ws-java-client/src/main/java/org/janelia/render/client/tile/RenderTilesClient.java b/render-ws-java-client/src/main/java/org/janelia/render/client/tile/RenderTilesClient.java index 6e5327e1f..f63d70b06 100644 --- a/render-ws-java-client/src/main/java/org/janelia/render/client/tile/RenderTilesClient.java +++ b/render-ws-java-client/src/main/java/org/janelia/render/client/tile/RenderTilesClient.java @@ -304,33 +304,33 @@ public void setupHackStackAsNeeded() public void setupStorageDirectories() throws IOException { - for (final double z : renderDataClient.getStackZValues(stack)) { + LOG.info("setupStorageDirectories: entry"); - final List relativePathSegments = getImageParentPathSegments(z); - final URI parentUri = storageBackend.resolvePath(relativePathSegments); + if (storageBackend instanceof CloudStorage) { - LOG.info("setupStorageDirectories: ensuring writable directory (or bucket 'prefix') for z {} at {}", - z, parentUri); - try { - storageBackend.ensureWritableDirectory(parentUri); - - final String scheme = storageBackend.root.getScheme(); - if ((scheme != null) && scheme.startsWith("gs:")) { - // Supposedly, ~1000 object mutations per second per bucket per project are allowed by Google. - // We have seen "exceeded the rate limit for object mutation operations" exceptions even after - // limiting the rate to 20 per second. - // Limiting the rate to 10 per second to try to avoid this. - Thread.sleep(100); - } + LOG.info("setupStorageDirectories: skipping since there is no need to set anything up for CloudStorage"); + + } else { + + for (final double z : renderDataClient.getStackZValues(stack)) { + + final List relativePathSegments = getImageParentPathSegments(z); + final URI parentUri = storageBackend.resolvePath(relativePathSegments); + + LOG.info("setupStorageDirectories: ensuring writable directory for z {} at {}", z, parentUri); - } catch (final Throwable t) { - LOG.warn("setupStorageDirectories: caught exception and will retry setup in 5 seconds", t); try { - Thread.sleep(5000); - } catch (final InterruptedException ie) { - LOG.warn("setupStorageDirectories: caught exception while sleeping and will retry setup now", ie); + storageBackend.ensureWritableDirectory(parentUri); + } catch (final Throwable t) { + LOG.warn("setupStorageDirectories: caught exception and will retry setup in 5 seconds", t); + try { + Thread.sleep(5000); + } catch (final InterruptedException ie) { + LOG.warn("setupStorageDirectories: caught exception while sleeping and will retry setup now", ie); + } + storageBackend.ensureWritableDirectory(parentUri); } - storageBackend.ensureWritableDirectory(parentUri); + } } @@ -656,13 +656,7 @@ private static class CloudStorage extends StorageBackend { @Override void ensureWritableDirectory(final URI uri) { - if (!keyValueAccess.exists(uri.getPath())) { - try { - keyValueAccess.createDirectories(uri.getPath()); - } catch (final IOException e) { - throw new RuntimeException("Could not create directory " + uri, e); - } - } + // no-op for cloud storage } @Override diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/ClusterCountClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/ClusterCountClientTest.java index 98e4608df..1de17219e 100644 --- a/render-ws-java-client/src/test/java/org/janelia/render/client/ClusterCountClientTest.java +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/ClusterCountClientTest.java @@ -41,8 +41,8 @@ public static void main(final String[] args) // "--maxSmallClusterSizeToSave", "100" }; - ClusterCountClient.main(effectiveArgs); -// findWafer61Clusters(); +// ClusterCountClient.main(effectiveArgs); + findWafer61Clusters(); } @@ -53,9 +53,9 @@ public static void findWafer61Clusters() throws Exception { final String baseDataUrl = "http://em-services-1:8080/render-ws/v1"; final String owner = "hess_wafers_60_61"; - final String project = "w61_serial_100_to_109"; - final String stack = "w61_s108_r00_gc_par"; - final int lastZ = 96; + final String project = "w61_serial_170_to_179"; + final String stack = "w61_s171_r01_gc_icc_pa_mat_render"; + final int lastZ = 82; final String mcName = stack + "_match"; final StackId stackId = new StackId(owner, project, stack); @@ -71,10 +71,9 @@ public static void findWafer61Clusters() throws Exception { jcccp.multiProject = MultiProjectParameters.singleStackInstance(baseDataUrl, stackId); jcccp.tileCluster = new TileClusterParameters(); - final int zCount = 96; jcccp.tileCluster.maxSmallClusterSize = 0; jcccp.tileCluster.includeMatchesOutsideGroup = true; - jcccp.tileCluster.maxLayersPerBatch = zCount + 1; + jcccp.tileCluster.maxLayersPerBatch = lastZ + 1; jcccp.tileCluster.maxOverlapLayers = 6; final RenderDataClient dataClient = new RenderDataClient(baseDataUrl, owner, project); diff --git a/render-ws-java-client/src/test/java/org/janelia/render/client/TilePairClientTest.java b/render-ws-java-client/src/test/java/org/janelia/render/client/TilePairClientTest.java index c5888ec88..39a46226f 100644 --- a/render-ws-java-client/src/test/java/org/janelia/render/client/TilePairClientTest.java +++ b/render-ws-java-client/src/test/java/org/janelia/render/client/TilePairClientTest.java @@ -205,15 +205,19 @@ public TileBoundsRTree buildRTree(final double z) { // public static void main(final String[] args) { // // final String[] effectiveArgs = (args != null) && (args.length > 0) ? args : new String[] { -// "--baseDataUrl", "http://tem-services.int.janelia.org:8080/render-ws/v1", -// "--owner", "flyTEM", -// "--project", "FAFB_montage", -// "--stack", "check_923_split_rough", +// "--baseDataUrl", "http://renderer-dev.int.janelia.org:8080/render-ws/v1", +// "--owner", "hess_wafers_60_61", +// "--project", "w61_serial_140_to_149", +// "--stack", "w61_s140_r00_gc", // "--xyNeighborFactor", "0.6", // "--excludeCornerNeighbors", "false", -// "--excludeSameLayerNeighbors", "true", +// "--excludeSameLayerNeighbors", "false", +// "--excludeSameSectionNeighbors", "false", +// "--excludeSameMfovNeighbors", "false", +// "--excludeDifferentMfovNeighbors", "true", // "--excludeCompletelyObscuredTiles", "false", -// "--zNeighborDistance", "40", +// "--zNeighborDistance", "0", +//// "--z", "1", "2", "3", // "--toJson", "/Users/trautmane/Desktop/test_pairs.json" // }; // TilePairClient.main(effectiveArgs); diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/BeamCorrectionSparkClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/BeamCorrectionSparkClient.java new file mode 100644 index 000000000..0837d66d5 --- /dev/null +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/BeamCorrectionSparkClient.java @@ -0,0 +1,151 @@ +package org.janelia.render.client.spark.multisem; + +import com.beust.jcommander.ParametersDelegate; + +import java.io.IOException; +import java.io.Serializable; +import java.util.List; + +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.janelia.alignment.spec.stack.StackId; +import org.janelia.alignment.spec.stack.StackIdNamingGroup; +import org.janelia.render.client.ClientRunner; +import org.janelia.render.client.RenderDataClient; +import org.janelia.render.client.multisem.BeamCorrectionClient; +import org.janelia.render.client.parameter.BeamCorrectionParameters; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.MultiProjectParameters; +import org.janelia.render.client.spark.LogUtilities; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineParameters; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineStep; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineStepId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Spark client for applying the pre-computed degree-0 beam-homogenization correction to multi-SEM stacks. + * + * @see BeamCorrectionClient + */ +public class BeamCorrectionSparkClient + implements Serializable, AlignmentPipelineStep { + + public static class Parameters extends CommandLineParameters { + + @ParametersDelegate + public MultiProjectParameters multiProject = new MultiProjectParameters(); + + @ParametersDelegate + public BeamCorrectionParameters beamCorrection = new BeamCorrectionParameters(); + } + + public static void main(final String[] args) { + final ClientRunner clientRunner = new ClientRunner(args) { + @Override + public void runClient(final String[] args) throws Exception { + final Parameters parameters = new Parameters(); + parameters.parse(args); + parameters.beamCorrection.validate(); + + LOG.info("runClient: entry, parameters={}", parameters); + + final BeamCorrectionSparkClient client = new BeamCorrectionSparkClient(); + client.createContextAndRun(parameters); + } + }; + clientRunner.run(); + } + + public BeamCorrectionSparkClient() { + } + + /** + * Create a spark context and run the client with the specified parameters. + */ + public void createContextAndRun(final Parameters clientParameters) + throws IOException { + final SparkConf conf = new SparkConf().setAppName(getClass().getSimpleName()); + try (final JavaSparkContext sparkContext = new JavaSparkContext(conf)) { + + LOG.info("createContextAndRun: appId is {}", sparkContext.getConf().getAppId()); + + correctBeamIntensity(sparkContext, + clientParameters.multiProject, + clientParameters.beamCorrection); + } + } + + /** Validates the specified pipeline parameters are sufficient. */ + @Override + public void validatePipelineParameters(final AlignmentPipelineParameters pipelineParameters) + throws IllegalArgumentException { + final BeamCorrectionParameters beamCorrection = pipelineParameters.getBeamCorrection(); + AlignmentPipelineParameters.validateRequiredElementExists("beamCorrection", + beamCorrection); + beamCorrection.validate(); + } + + /** Run the client as part of an alignment pipeline. */ + @Override + public void runPipelineStep(final JavaSparkContext sparkContext, + final AlignmentPipelineParameters pipelineParameters) + throws IllegalArgumentException, IOException { + + final StackIdNamingGroup rawNamingGroup = pipelineParameters.getRawNamingGroup(); + if (rawNamingGroup == null) { + throw new IllegalArgumentException( + "The " + AlignmentPipelineStepId.CORRECT_BEAM_INTENSITY + " pipeline step requires that " + + "a 'raw' pipelineStackGroup is defined in the pipeline parameters."); + } + + final MultiProjectParameters multiProject = pipelineParameters.getMultiProject(rawNamingGroup); + + correctBeamIntensity(sparkContext, + multiProject, + pipelineParameters.getBeamCorrection()); + } + + @Override + public AlignmentPipelineStepId getDefaultStepId() { + return AlignmentPipelineStepId.CORRECT_BEAM_INTENSITY; + } + + public void correctBeamIntensity(final JavaSparkContext sparkContext, + final MultiProjectParameters multiProject, + final BeamCorrectionParameters beamCorrection) throws IOException { + + final String baseDataUrl = multiProject.getBaseDataUrl(); + + // all z values of each stack are corrected, so only the stack ids are needed here + final List stackIdList = multiProject.stackIdWithZ.getStackIdList(multiProject.getDataClient()); + + LOG.info("correctBeamIntensity: entry, distributing {} stacks with beamCorrection {}", + stackIdList.size(), beamCorrection); + + if (stackIdList.isEmpty()) { + throw new IllegalArgumentException("no stacks match the specified parameters"); + } + + final JavaRDD rddStackIds = sparkContext.parallelize(stackIdList); + + rddStackIds.foreach(stackId -> { + + LogUtilities.setupExecutorLog4j(stackId.toDevString()); + + final RenderDataClient executorDataClient = new RenderDataClient(baseDataUrl, + stackId.getOwner(), + stackId.getProject()); + + new BeamCorrectionClient().correctStack(executorDataClient, + stackId.getStack(), + beamCorrection, + true); + }); + + LOG.info("correctBeamIntensity: exit, corrected {} stacks", stackIdList.size()); + } + + private static final Logger LOG = LoggerFactory.getLogger(BeamCorrectionSparkClient.class); +} diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/CreepCorrectionSparkClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/CreepCorrectionSparkClient.java index 462751b89..e0d3c2cfd 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/CreepCorrectionSparkClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/CreepCorrectionSparkClient.java @@ -19,6 +19,7 @@ import org.janelia.alignment.match.MatchCollectionId; import org.janelia.alignment.spec.ResolvedTileSpecCollection; import org.janelia.alignment.spec.stack.StackId; +import org.janelia.alignment.spec.stack.StackIdNamingGroup; import org.janelia.alignment.spec.stack.StackMetaData; import org.janelia.alignment.spec.stack.StackWithZValues; import org.janelia.render.client.ClientRunner; @@ -119,9 +120,15 @@ public void validatePipelineParameters(final AlignmentPipelineParameters pipelin public void runPipelineStep(final JavaSparkContext sparkContext, final AlignmentPipelineParameters pipelineParameters) throws IllegalArgumentException, IOException { - - final MultiProjectParameters multiProject = - pipelineParameters.getMultiProject(pipelineParameters.getRawNamingGroup()); + + final StackIdNamingGroup rawNamingGroup = pipelineParameters.getRawNamingGroup(); + if (rawNamingGroup == null) { + throw new IllegalArgumentException( + "The " + AlignmentPipelineStepId.CORRECT_CREEP + " pipeline step requires that " + + "a 'raw' pipelineStackGroup is defined in the pipeline parameters."); + } + + final MultiProjectParameters multiProject = pipelineParameters.getMultiProject(rawNamingGroup); for (final StackWithZValues stackWithAllZ : multiProject.buildListOfStackWithAllZ()) { correctCreep(sparkContext, @@ -151,8 +158,6 @@ public void correctCreep(final JavaSparkContext sparkContext, final String sourceStack = sourceStackId.getStack(); final String targetStack = creepCorrection.getTargetStack(sourceStack); - final MatchCollectionId targetMatchCollectionId = new MatchCollectionId(matchCollectionId.getOwner(), - targetStack + "_match"); final RenderDataClient sourceDataClient = new RenderDataClient(baseDataUrl, sourceStackId.getOwner(), @@ -214,8 +219,45 @@ public void correctCreep(final JavaSparkContext sparkContext, sourceDataClient.setStackState(targetStack, StackMetaData.StackState.COMPLETE); // Phase 2: transform matches - if (! creepCorrection.skipMatchCorrection) { - transformMatches(sparkContext, baseDataUrl, matchCollectionId, targetMatchCollectionId, allResults); + if (creepCorrection.correctMatches()) { + + final RenderDataClient driverMatchClient = new RenderDataClient(baseDataUrl, + matchCollectionId.getOwner(), + matchCollectionId.getName()); + + final MatchCollectionId targetMatchCollectionId = new MatchCollectionId(matchCollectionId.getOwner(), + targetStack + "_match"); + + if (creepCorrection.overwriteMatchData()) { + + transformMatches(sparkContext, + driverMatchClient, + baseDataUrl, + matchCollectionId, + matchCollectionId, + allResults); + + } else if (creepCorrection.overwriteMatchDataAndRenameAsTarget()) { + + transformMatches(sparkContext, + driverMatchClient, + baseDataUrl, + matchCollectionId, + matchCollectionId, + allResults); + + driverMatchClient.renameMatchCollection(targetMatchCollectionId); + + } else if (creepCorrection.writeMatchDataToTargetCollection()) { + + transformMatches(sparkContext, + driverMatchClient, + baseDataUrl, + matchCollectionId, + targetMatchCollectionId, + allResults); + } + } else { LOG.info("correctCreep: skipping match correction"); } @@ -224,16 +266,13 @@ public void correctCreep(final JavaSparkContext sparkContext, } private void transformMatches(final JavaSparkContext sparkContext, + final RenderDataClient driverMatchClient, final String baseDataUrl, final MatchCollectionId matchCollectionId, final MatchCollectionId targetMatchCollectionId, final Map> allResults) throws IOException { - final RenderDataClient driverMatchClient = new RenderDataClient(baseDataUrl, - matchCollectionId.getOwner(), - matchCollectionId.getName()); - final List pGroupIds = driverMatchClient.getMatchPGroupIds(); if (pGroupIds.isEmpty()) { diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/LayerAsTileClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/LayerAsTileClient.java index 559baaafe..e5edbe58c 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/LayerAsTileClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/LayerAsTileClient.java @@ -154,7 +154,7 @@ private static void buildDynamicLayerAsTileStacks(final JavaSparkContext sparkCo final List align2DSfovStacksWithAllZ = layerAsTileStackLists.getAlign2DSfovStacksWithAllZ(); - final int parallelism = Math.min(MFOVASTileClient.MAX_PARTITIONS_FOR_ONE_WEB_SERVER, align2DSfovStacksWithAllZ.size()); + final int parallelism = Math.min(MFOVAsTileClient.MAX_PARTITIONS_FOR_ONE_WEB_SERVER, align2DSfovStacksWithAllZ.size()); LOG.info("buildDynamicLayerAsTileStacks: entry, distributing build of {} stack(s) with parallelism {} (defaultParallelism={})", align2DSfovStacksWithAllZ.size(), parallelism, sparkContext.defaultParallelism()); @@ -252,7 +252,7 @@ private static void buildRenderedLayerAsTileStacks(final JavaSparkContext sparkC if (! layerClientInfoList.isEmpty()) { - final int parallelism = Math.min(MFOVASTileClient.MAX_PARTITIONS_FOR_ONE_WEB_SERVER, layerClientInfoList.size()); + final int parallelism = Math.min(MFOVAsTileClient.MAX_PARTITIONS_FOR_ONE_WEB_SERVER, layerClientInfoList.size()); LOG.info("buildRenderedLayerAsTileStacks: distributing rendering for {} layers with parallelism {} (defaultParallelism={})", layerClientInfoList.size(), parallelism, sparkContext.defaultParallelism()); @@ -409,7 +409,7 @@ private static void buildAlign3DSfovStacks(final JavaSparkContext sparkContext, if (! rawSfovStacksNeedingAlign3DStack.isEmpty()) { - final int parallelism = Math.min(MFOVASTileClient.MAX_PARTITIONS_FOR_ONE_WEB_SERVER, rawSfovStacksNeedingAlign3DStack.size()); + final int parallelism = Math.min(MFOVAsTileClient.MAX_PARTITIONS_FOR_ONE_WEB_SERVER, rawSfovStacksNeedingAlign3DStack.size()); LOG.info("buildAlign3DSfovStacks: distributing build of {} stack(s) with parallelism {} (defaultParallelism={})", rawSfovStacksNeedingAlign3DStack.size(), parallelism, sparkContext.defaultParallelism()); diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVASTileClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVAsTileClient.java similarity index 95% rename from render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVASTileClient.java rename to render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVAsTileClient.java index 26d416774..79dab3ac7 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVASTileClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVAsTileClient.java @@ -29,6 +29,7 @@ import org.janelia.alignment.spec.ResolvedTileSpecCollection.TransformApplicationMethod; import org.janelia.alignment.spec.TileSpec; import org.janelia.alignment.spec.stack.StackId; +import org.janelia.alignment.spec.stack.StackIdNamingGroup; import org.janelia.alignment.spec.stack.StackMetaData; import org.janelia.alignment.spec.stack.StackWithZValues; import org.janelia.render.client.ClientRunner; @@ -60,7 +61,7 @@ /** * Spark client for ... */ -public class MFOVASTileClient +public class MFOVAsTileClient implements Serializable, AlignmentPipelineStep { public static class Parameters extends CommandLineParameters { @@ -87,7 +88,7 @@ public static void main(final String[] args) { public void runClient(final String[] args) throws Exception { final Parameters parameters = new Parameters(); parameters.parse(args); - final MFOVASTileClient client = new MFOVASTileClient(); + final MFOVAsTileClient client = new MFOVAsTileClient(); client.createContextAndRun(parameters); } }; @@ -95,7 +96,7 @@ public void runClient(final String[] args) throws Exception { } /** Empty constructor required for alignment pipeline steps. */ - public MFOVASTileClient() { + public MFOVAsTileClient() { } /** Create a spark context and run the client with the specified parameters. */ @@ -120,7 +121,15 @@ public void runPipelineStep(final JavaSparkContext sparkContext, final AlignmentPipelineParameters pipelineParameters) throws IllegalArgumentException, IOException { final Parameters clientParameters = new Parameters(); - clientParameters.multiProject = pipelineParameters.getMultiProject(pipelineParameters.getRawNamingGroup()); + + final StackIdNamingGroup otherNamingGroup = pipelineParameters.getOtherNamingGroup(); + if (otherNamingGroup == null) { + throw new IllegalArgumentException( + "The " + AlignmentPipelineStepId.MFOV_AS_TILE + " pipeline step requires that " + + "an 'other' pipelineStackGroup is defined in the pipeline parameters."); + } + + clientParameters.multiProject = pipelineParameters.getMultiProject(otherNamingGroup); clientParameters.mfovAsTile = pipelineParameters.getMfovAsTile(); run(sparkContext, clientParameters); } @@ -142,7 +151,9 @@ private void run(final JavaSparkContext sparkContext, clientParameters.multiProject, clientParameters.mfovAsTile); if (clientParameters.mfovAsTile.doPrealign()) { - alignAndIntensityCorrectMfovAsTileStacks(sparkContext, mfovAsTileStackLists); + alignAndIntensityCorrectMfovAsTileStacks(sparkContext, + mfovAsTileStackLists, + clientParameters.mfovAsTile.isDeriveSfovMatchData()); } buildDynamicMfovAsTileStacks(sparkContext, mfovAsTileStackLists); @@ -163,8 +174,10 @@ private void run(final JavaSparkContext sparkContext, } private static void alignAndIntensityCorrectMfovAsTileStacks(final JavaSparkContext sparkContext, - final MFOVAsTileStackLists mfovAsTileStackLists) + final MFOVAsTileStackLists mfovAsTileStackLists, + final boolean deriveSfovMatchData) throws IOException { + LOG.info("alignAndIntensityCorrectMfovAsTileStacks: entry"); final String baseDataUrl = mfovAsTileStackLists.getBaseDataUrl(); @@ -177,7 +190,10 @@ private static void alignAndIntensityCorrectMfovAsTileStacks(final JavaSparkCont for (final StackWithZValues rawSfovStackWithZ : rawSfovStacksWithAllZ) { final StackId rawSfovStackId = rawSfovStackWithZ.getStackId(); + final String rawSfovStack = rawSfovStackId.getStack(); final StackId prealignedStackId = rawSfovStackId.withStackSuffix(prealignedSfovStackSuffix); + final MatchCollectionId rawSfovMatchCollectionId = deriveSfovMatchData ? null : new MatchCollectionId(rawSfovStackId.getOwner(), + rawSfovStack + "_match"); if (mfovAsTileStackLists.isExistingStack(prealignedStackId)) { LOG.info("alignAndIntensityCorrectMfovAsTileStacks: skipping build of {} because it already exists", @@ -186,23 +202,24 @@ private static void alignAndIntensityCorrectMfovAsTileStacks(final JavaSparkCont } // Create prealigned stack - final RenderDataClient dataClient = new RenderDataClient(baseDataUrl, - rawSfovStackId.getOwner(), - rawSfovStackId.getProject()); - final StackMetaData rawStackMetaData = dataClient.getStackMetaData(rawSfovStackId.getStack()); - dataClient.setupDerivedStack(rawStackMetaData, prealignedStackId.getStack()); + final RenderDataClient stackDataClient = new RenderDataClient(baseDataUrl, + rawSfovStackId.getOwner(), + rawSfovStackId.getProject()); + final StackMetaData rawStackMetaData = stackDataClient.getStackMetaData(rawSfovStackId.getStack()); + stackDataClient.setupDerivedStack(rawStackMetaData, prealignedStackId.getStack()); prealignedStackIds.add(prealignedStackId); // Collect MFOV tasks for each layer for (final Double z : rawSfovStackWithZ.getzValues()) { - final List mfovNames = MultiProjectParameters.getSortedMFOVNamesForOneLayer(dataClient, - rawSfovStackId.getStack(), + final List mfovNames = MultiProjectParameters.getSortedMFOVNamesForOneLayer(stackDataClient, + rawSfovStack, z); for (final String mfovName : mfovNames) { mfovTasks.add(new MfovPrealignTask(baseDataUrl, - rawSfovStackId, - prealignedStackId, - new LayerMFOV(z, mfovName)) + rawSfovStackId, + prealignedStackId, + new LayerMFOV(z, mfovName), + rawSfovMatchCollectionId) ); } } @@ -583,8 +600,6 @@ private static void alignRenderedMfovAsTileStacks(final JavaSparkContext sparkCo final MFOVAsTileParameters mfovAsTile = mfovAsTileStackLists.getMfovAsTile(); final AffineBlockSolverSetup translationSetup = mfovAsTile.buildMfovAffineBlockSolverSetup(MFOVAsTileParameters.SolveType.TRANSLATION); - final AffineBlockSolverSetup affineSetup = - mfovAsTile.buildMfovAffineBlockSolverSetup(MFOVAsTileParameters.SolveType.AFFINE); final boolean deriveMatchCollectionNamesFromProject = false; // use standard stack-based match collection names final String matchSuffix = ""; // without any suffix @@ -604,10 +619,6 @@ private static void alignRenderedMfovAsTileStacks(final JavaSparkContext sparkCo renderedMfovStackWithAllZ, deriveMatchCollectionNamesFromProject, matchSuffix)); - setupList.add(affineSetup.buildPipelineClone(baseDataUrl, - renderedMfovStackWithAllZ, - deriveMatchCollectionNamesFromProject, - matchSuffix)); } } @@ -839,5 +850,5 @@ public int renderTiles() public static final int MAX_PARTITIONS_FOR_ONE_WEB_SERVER = 100000; - private static final Logger LOG = LoggerFactory.getLogger(MFOVASTileClient.class); + private static final Logger LOG = LoggerFactory.getLogger(MFOVAsTileClient.class); } diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVMontageMatchPatchClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVMontageMatchPatchClient.java index 8587e55d4..bcd741f77 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVMontageMatchPatchClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MFOVMontageMatchPatchClient.java @@ -104,6 +104,7 @@ public void createContextAndRun(final Parameters clientParameters) throws IOExce if (patchParameters.checkLayerConnectedClusters) { checkLayerConnectedClusters(sparkContext, clientParameters.multiProject, + patchParameters, tileClusterParameters); } @@ -135,7 +136,10 @@ public void runPipelineStep(final JavaSparkContext sparkContext, patchMFOVs(sparkContext, multiProjectParameters, patchParameters); if (patchParameters.checkLayerConnectedClusters) { - checkLayerConnectedClusters(sparkContext, multiProjectParameters, pipelineParameters.getTileCluster()); + checkLayerConnectedClusters(sparkContext, + multiProjectParameters, + patchParameters, + pipelineParameters.getTileCluster()); } } @@ -195,7 +199,7 @@ private void patchIsolatedMFOVs(final JavaSparkContext sparkContext, false, renderDataClient, patchParameters.addIsolatedEdgeLabel, - patchParameters.resinMfovStartPositionMatchWeight); + patchParameters.getResinMfovStartPositionMatchWeight()); return null; }; @@ -319,6 +323,7 @@ private void completeTrimStacks(final MultiProjectParameters multiProject) private void checkLayerConnectedClusters(final JavaSparkContext sparkContext, final MultiProjectParameters multiProjectParameters, + final MFOVMontageMatchPatchParameters patchParameters, final TileClusterParameters tileClusterParameters) throws IOException { @@ -331,7 +336,6 @@ private void checkLayerConnectedClusters(final JavaSparkContext sparkContext, final List connectedTileClusterSummaryForStacks = clusterCountClient.findConnectedClusters(sparkContext, clientParameters); - final List stackErrors = new ArrayList<>(); for (final ConnectedTileClusterSummaryForStack summary : connectedTileClusterSummaryForStacks) { @@ -339,7 +343,25 @@ private void checkLayerConnectedClusters(final JavaSparkContext sparkContext, final RenderDataClient renderDataClient = multiProjectParameters.getDataClient().buildClient(stackId.getOwner(), stackId.getProject()); final List zValues = renderDataClient.getStackZValues(stackId.getStack()); - final String countErrorString = summary.buildCountErrorString(zValues.size(), + + int expectedTileClusterCount = zValues.size(); + if (! patchParameters.isIsolatedMfovPatchingNeeded()) { + // resin MFOV patching derives the only cross MFOV matches (isolated edge labelling derives none), + // so without it the expected cluster count is the total number of layer MFOVs + expectedTileClusterCount = 0; + for (final Double z : zValues) { + final List sortedMFOVNames = + MultiProjectParameters.getSortedMFOVNamesForOneLayer(renderDataClient, + stackId.getStack(), + z); + expectedTileClusterCount += sortedMFOVNames.size(); + } + } + + LOG.info("checkLayerConnectedClusters: expectedTileClusterCount is {} for {}", + expectedTileClusterCount, stackId.toDevString()); + + final String countErrorString = summary.buildCountErrorString(expectedTileClusterCount, 0, 0); if (! countErrorString.isEmpty()) { diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MatchCollectionRenameClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MatchCollectionRenameClient.java new file mode 100644 index 000000000..e760595e0 --- /dev/null +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MatchCollectionRenameClient.java @@ -0,0 +1,154 @@ +package org.janelia.render.client.spark.multisem; + +import com.beust.jcommander.ParametersDelegate; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.apache.spark.api.java.JavaSparkContext; +import org.janelia.alignment.match.MatchCollectionId; +import org.janelia.render.client.ClientRunner; +import org.janelia.render.client.RenderDataClient; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.MatchCollectionRenameParameters; +import org.janelia.render.client.parameter.MultiProjectParameters; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineParameters; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineStep; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineStepId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Client for renaming an owner's match collections (e.g. so that collection names stay derived from + * the names of the stacks that later pipeline steps use). + * + *

    Renames are quick web service calls, so nothing is distributed to spark workers here.

    + */ +public class MatchCollectionRenameClient + implements Serializable, AlignmentPipelineStep { + + public static class Parameters extends CommandLineParameters { + + @ParametersDelegate + public MultiProjectParameters multiProject = new MultiProjectParameters(); + + @ParametersDelegate + public MatchCollectionRenameParameters matchCollectionRename = new MatchCollectionRenameParameters(); + } + + public static void main(final String[] args) { + final ClientRunner clientRunner = new ClientRunner(args) { + @Override + public void runClient(final String[] args) throws Exception { + final Parameters parameters = new Parameters(); + parameters.parse(args); + parameters.matchCollectionRename.validate(); + + LOG.info("runClient: entry, parameters={}", parameters); + + // NOTE: no spark context is needed here because all renames are run on the driver + final MatchCollectionRenameClient client = new MatchCollectionRenameClient(); + client.renameMatchCollections(parameters.multiProject.getBaseDataUrl(), + parameters.matchCollectionRename); + } + }; + clientRunner.run(); + } + + public MatchCollectionRenameClient() { + } + + /** Validates the specified pipeline parameters are sufficient. */ + @Override + public void validatePipelineParameters(final AlignmentPipelineParameters pipelineParameters) + throws IllegalArgumentException { + final MatchCollectionRenameParameters matchCollectionRename = pipelineParameters.getMatchCollectionRename(); + AlignmentPipelineParameters.validateRequiredElementExists("matchCollectionRename", + matchCollectionRename); + matchCollectionRename.validate(); + } + + /** Run the client as part of an alignment pipeline. */ + @Override + public void runPipelineStep(final JavaSparkContext sparkContext, + final AlignmentPipelineParameters pipelineParameters) + throws IllegalArgumentException, IOException { + + final MultiProjectParameters multiProject = pipelineParameters.getMultiProject(null); + + renameMatchCollections(multiProject.getBaseDataUrl(), + pipelineParameters.getMatchCollectionRename()); + } + + @Override + public AlignmentPipelineStepId getDefaultStepId() { + return AlignmentPipelineStepId.RENAME_MATCH_COLLECTIONS; + } + + public void renameMatchCollections(final String baseDataUrl, + final MatchCollectionRenameParameters matchCollectionRename) + throws IllegalArgumentException, IOException { + + LOG.info("renameMatchCollections: entry, matchCollectionRename={}", matchCollectionRename); + + final String owner = matchCollectionRename.matchCollectionOwner; + final RenderDataClient ownerDataClient = new RenderDataClient(baseDataUrl, owner, "not_used"); + + final List existingNames = ownerDataClient.getOwnerMatchCollections().stream() + .map(mcmd -> mcmd.getCollectionId().getName()) + .sorted() + .collect(Collectors.toList()); + + final Pattern sourcePattern = matchCollectionRename.buildSourceNamePattern(); + + // sourceNames and targetNames are parallel lists, so targetNames.get(i) is the new name for sourceNames.get(i) + final List sourceNames = new ArrayList<>(); + final List targetNames = new ArrayList<>(); + for (final String existingName : existingNames) { + final Matcher matcher = sourcePattern.matcher(existingName); + if (matcher.matches()) { + sourceNames.add(existingName); + targetNames.add(matcher.replaceFirst(matchCollectionRename.targetNamePattern)); + } + } + + // check all renames before doing any of them so that partial renames are less likely + final Set existingNameSet = new HashSet<>(existingNames); + final Set distinctTargetNames = new HashSet<>(); + for (int i = 0; i < targetNames.size(); i++) { + final String targetName = targetNames.get(i); + if (existingNameSet.contains(targetName)) { + throw new IllegalArgumentException("cannot rename " + sourceNames.get(i) + " to " + targetName + + " because a collection with that name already exists for owner " + + owner); + } + if (! distinctTargetNames.add(targetName)) { + throw new IllegalArgumentException("cannot rename more than one collection to " + targetName + + " for owner " + owner); + } + } + + if (targetNames.isEmpty()) { + LOG.warn("renameMatchCollections: none of the {} collection(s) for owner {} match the sourceNamePattern '{}'", + existingNames.size(), owner, matchCollectionRename.sourceNamePattern); + } + + for (int i = 0; i < targetNames.size(); i++) { + final String sourceName = sourceNames.get(i); + final MatchCollectionId targetCollectionId = new MatchCollectionId(owner, targetNames.get(i)); + final RenderDataClient matchDataClient = new RenderDataClient(baseDataUrl, owner, sourceName); + matchDataClient.renameMatchCollection(targetCollectionId); + } + + LOG.info("renameMatchCollections: exit, renamed {} collection(s)", targetNames.size()); + } + + private static final Logger LOG = LoggerFactory.getLogger(MatchCollectionRenameClient.class); +} diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MfovPrealignTask.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MfovPrealignTask.java index 06910d8ec..b011487e5 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MfovPrealignTask.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/multisem/MfovPrealignTask.java @@ -10,6 +10,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import mpicbg.imagefeatures.Feature; import mpicbg.models.ErrorStatistic; @@ -28,14 +29,18 @@ import org.janelia.alignment.match.CanvasFeatureMatcher; import org.janelia.alignment.match.CanvasId; import org.janelia.alignment.match.CanvasMatchResult; +import org.janelia.alignment.match.CanvasMatches; +import org.janelia.alignment.match.MatchCollectionId; import org.janelia.alignment.match.MatchFilter; import org.janelia.alignment.match.ModelType; import org.janelia.alignment.match.OrderedCanvasIdPair; import org.janelia.alignment.match.parameters.FeatureExtractionParameters; import org.janelia.alignment.match.parameters.MatchDerivationParameters; import org.janelia.alignment.multisem.LayerMFOV; +import org.janelia.alignment.spec.Bounds; import org.janelia.alignment.spec.LeafTransformSpec; import org.janelia.alignment.spec.ResolvedTileSpecCollection; +import org.janelia.alignment.spec.ResolvedTileSpecsWithMatchPairs; import org.janelia.alignment.spec.TileSpec; import org.janelia.alignment.spec.stack.StackId; import org.janelia.alignment.util.ImageProcessorCache; @@ -58,6 +63,7 @@ public class MfovPrealignTask implements Serializable { private final StackId rawSfovStackId; private final StackId prealignedStackId; private final LayerMFOV layerMfov; + private final MatchCollectionId matchCollectionId; private static final int CLIP_SIZE = 150; @@ -65,12 +71,14 @@ public MfovPrealignTask( final String baseDataUrl, final StackId rawSfovStackId, final StackId prealignedStackId, - final LayerMFOV layerMfov + final LayerMFOV layerMfov, + final MatchCollectionId matchCollectionId ) { this.baseDataUrl = baseDataUrl; this.rawSfovStackId = rawSfovStackId; this.prealignedStackId = prealignedStackId; this.layerMfov = layerMfov; + this.matchCollectionId = matchCollectionId; } public String toString() { @@ -94,31 +102,22 @@ public void run() rawSfovStackId.getOwner(), rawSfovStackId.getProject()); - // 1. Fetch tile specs for all SFOVs in this MFOV - final ResolvedTileSpecCollection mfovTiles = fetchMfovTileSpecs(dataClient); - - if (mfovTiles.getTileCount() == 0) { - throw new IOException("no tile specs found for " + this); - } - - // 2. Generate tile pairs for matching - final List tilePairs = generateTilePairs(mfovTiles); - - if (tilePairs.isEmpty()) { - LOG.info("run: exit, {}, no tile pairs generated", this); - return; - } - // Initialize the image processor cache with ~1Gb size to hold all tiles in memory - final ImageProcessorCache cache = new ImageProcessorCache(1_000_000_000L, false, false); + final ImageProcessorCache cache = new ImageProcessorCache(1_000_000_000L, + false, + false); - // 3. Align tiles within this MFOV - final ResolvedTileSpecCollection alignedTiles = alignTiles(mfovTiles, tilePairs, cache); + final ResolvedTileSpecsWithMatchPairs sfovTilesAndMatchesForMfov = fetchSfovTileSpecsWithMatchPairs(dataClient, + cache); + final ResolvedTileSpecCollection alignedTiles = alignTiles(sfovTilesAndMatchesForMfov); - // 4. Perform intensity correction - final ResolvedTileSpecCollection alignedIcTiles = intensityCorrectTiles(alignedTiles, tilePairs, cache); + final List tilePairs = sfovTilesAndMatchesForMfov.getMatchPairs().stream() + .map(CanvasMatches::toOrderedPair) + .collect(Collectors.toList()); + final ResolvedTileSpecCollection alignedIcTiles = intensityCorrectTiles(alignedTiles, + tilePairs, + cache); - // 5. Push the aligned tile specs to the prealigned stack dataClient.saveResolvedTiles(alignedIcTiles, prealignedStackId.getStack(), layerMfov.getZ()); final long elapsedSeconds = (System.currentTimeMillis() - startTime) / 1000; @@ -165,13 +164,73 @@ private void setupLogging() { } /** - * Fetch tile specs for all SFOVs belonging to this MFOV at the specified z layer. + * Fetch tile specs for all SFOVs belonging to this MFOV at the specified z layer + * and fetch (or build) the corresponding matches. */ - private ResolvedTileSpecCollection fetchMfovTileSpecs(final RenderDataClient dataClient) throws IOException { + private ResolvedTileSpecsWithMatchPairs fetchSfovTileSpecsWithMatchPairs(final RenderDataClient dataClient, + final ImageProcessorCache cache) + throws IOException { + + LOG.info("fetchSfovTileSpecsWithMatchPairs: entry, {}", this); + final String matchPattern = "_" + layerMfov.getSimpleMfovName() + "_"; // limit to tiles in this MFOV - return dataClient.getResolvedTiles(rawSfovStackId.getStack(), - layerMfov.getZ(), - matchPattern); + + final ResolvedTileSpecsWithMatchPairs resolvedTileSpecsWithMatchPairs; + if (matchCollectionId == null) { + + final ResolvedTileSpecCollection tiles = dataClient.getResolvedTiles(rawSfovStackId.getStack(), + layerMfov.getZ(), + matchPattern); + if (tiles.getTileCount() == 0) { + throw new IOException("no tile specs found for " + this); + } + + final List tilePairs = generateTilePairs(tiles); + if (tilePairs.isEmpty()) { + throw new IOException("no tile pairs generated for " + this); + } + + resolvedTileSpecsWithMatchPairs = generateAndAddMatches(tiles, + tilePairs, + cache); + } else { + + final Double z = layerMfov.getZ(); + final Bounds layerBounds = new Bounds(-Double.MAX_VALUE, -Double.MAX_VALUE, z, + Double.MAX_VALUE, Double.MAX_VALUE, z); + + resolvedTileSpecsWithMatchPairs = dataClient.getResolvedTilesWithMatchPairs(rawSfovStackId.getStack(), + layerBounds, + matchCollectionId.getName(), + null, + matchPattern, + false); + if (resolvedTileSpecsWithMatchPairs.getResolvedTileSpecs().getTileCount() == 0) { + throw new IOException("no tile specs found for " + this); + } + + final int originalMatchPairCount = resolvedTileSpecsWithMatchPairs.getMatchPairCount(); + resolvedTileSpecsWithMatchPairs.removeMatchPairsThatReferenceTilesOutsideThisCollection(); + final int normalizedMatchPairCount = resolvedTileSpecsWithMatchPairs.getMatchPairCount(); + if (normalizedMatchPairCount < originalMatchPairCount) { + LOG.info("fetchSfovTileSpecsWithMatchPairs: removed {} pairs and kept {} pairs for {}", + (originalMatchPairCount - normalizedMatchPairCount), normalizedMatchPairCount, this); + } + + // change matches to world coordinates to be consistent with original manual derivation approach + resolvedTileSpecsWithMatchPairs.changeMatchesToWorldCoordinates(); + } + + if (resolvedTileSpecsWithMatchPairs.getMatchPairCount() == 0) { + throw new IOException("no matches were found or built for " + this); + } + + LOG.info("fetchSfovTileSpecsWithMatchPairs: {} returning {} tiles and {} match pairs", + this, + resolvedTileSpecsWithMatchPairs.getResolvedTileSpecs().getTileCount(), + resolvedTileSpecsWithMatchPairs.getMatchPairCount()); + + return resolvedTileSpecsWithMatchPairs; } /** @@ -191,8 +250,8 @@ private List generateTilePairs(final ResolvedTileSpecCollec final Rectangle2D rectJ = new Rectangle2D.Double(tj.getMinX(), tj.getMinY(), tj.getWidth(), tj.getHeight()); if (rectI.intersects(rectJ)) { - final CanvasId canvasIdI = new CanvasId(ti.getGroupId(), ti.getTileId()); - final CanvasId canvasIdJ = new CanvasId(tj.getGroupId(), tj.getTileId()); + final CanvasId canvasIdI = new CanvasId(ti.getZ().toString(), ti.getTileId()); + final CanvasId canvasIdJ = new CanvasId(tj.getZ().toString(), tj.getTileId()); pairs.add(new OrderedCanvasIdPair(canvasIdI, canvasIdJ, 0.0)); } } @@ -204,56 +263,104 @@ private List generateTilePairs(final ResolvedTileSpecCollec /** * Perform SIFT feature matching on the tile pairs. */ - private ResolvedTileSpecCollection alignTiles( - final ResolvedTileSpecCollection tiles, - final List tilePairs, - final ImageProcessorCache cache - ) { + private ResolvedTileSpecsWithMatchPairs generateAndAddMatches(final ResolvedTileSpecCollection tiles, + final List tilePairs, + final ImageProcessorCache cache) { final long startTime = System.currentTimeMillis(); - LOG.info("alignTiles: entry, {}", this); + + LOG.info("generateAndAddMatches: entry, {}", this); + + final List matchPairs = new ArrayList<>(); // Extract features from all mfov tiles final CanvasFeatureExtractor featureExtractor = CanvasFeatureExtractor.build(FEATURE_EXTRACTION_PARAMETERS); final Map> mfovFeatures = new HashMap<>(tiles.getTileCount()); - final Map> modelTiles = new HashMap<>(); for (final TileSpec tileSpec : tiles.getTileSpecs()) { final List tileFeatures = extractBoundaryFeatures(cache, tileSpec, featureExtractor); - mfovFeatures.put(tileSpec.getTileId(), tileFeatures); - modelTiles.put(tileSpec.getTileId(), new Tile<>(new TranslationModel2D())); } // Match features between tile pairs - final CanvasFeatureMatcher featureMatcher = new CanvasFeatureMatcher(MATCH_DERIVATION_PARAMETERS, 1.0); + final double renderScale = 1.0; + final CanvasFeatureMatcher featureMatcher = new CanvasFeatureMatcher(MATCH_DERIVATION_PARAMETERS, + renderScale); + CanvasMatchResult matchResult; for (final OrderedCanvasIdPair pair : tilePairs) { - final String tileIdI = pair.getP().getId(); - final String tileIdJ = pair.getQ().getId(); - final CanvasMatchResult matchResult = featureMatcher.deriveMatchResult( - mfovFeatures.get(tileIdI), - mfovFeatures.get(tileIdJ) - ); + final CanvasId p = pair.getP(); + final CanvasId q = pair.getQ(); - // Connect tiles for optimization... - final Tile modelTileI = modelTiles.get(tileIdI); - final Tile modelTileJ = modelTiles.get(tileIdJ); + final String pTileId = p.getId(); + final String qTileId = q.getId(); + + matchResult = featureMatcher.deriveMatchResult(mfovFeatures.get(pTileId), + mfovFeatures.get(qTileId)); + + // create fake connection if no inliers found if (matchResult == null || matchResult.getTotalNumberOfInliers() == 0) { - // ... with fake matches if no inliers found - final PointMatch fakeMatch = new PointMatch( - new Point(new double[] {0.0, 0.0}), new Point(new double[] {0.0, 0.0}), 1e-4 - ); - modelTileI.connect(modelTileJ, Collections.singletonList(fakeMatch)); - } else { - // ... with real matches if inliers found - final List pointMatches = matchResult.getInlierPointMatchList(); - modelTileI.connect(modelTileJ, pointMatches); + final PointMatch fakeMatch = new PointMatch(new Point(new double[]{0.0, 0.0}), + new Point(new double[]{0.0, 0.0}), + 1e-4); + final List fakeList = new ArrayList<>(); + fakeList.add(fakeMatch); + final List> fakeInliers = new ArrayList<>(); + fakeInliers.add(fakeList); + matchResult = new CanvasMatchResult(null, fakeInliers, 1); } + + final List canvasMatchesList = matchResult.getInlierMatchesList(p.getGroupId(), + p.getId(), + q.getGroupId(), + q.getId(), + renderScale, + CanvasId.ZERO_OFFSETS, + CanvasId.ZERO_OFFSETS); + matchPairs.add(canvasMatchesList.get(0)); } - LOG.info("alignTiles: start optimization, {}", this); + final long elapsedSeconds = (System.currentTimeMillis() - startTime) / 1000; + + LOG.info("generateAndAddMatches: exit, {}, elapsedSeconds={}", this, elapsedSeconds); + + return new ResolvedTileSpecsWithMatchPairs(tiles, matchPairs); + } + + /** + * Align tile pairs. + */ + private ResolvedTileSpecCollection alignTiles(final ResolvedTileSpecsWithMatchPairs sfovTilesAndMatchesForMfov) { + + final long startTime = System.currentTimeMillis(); + LOG.info("alignTiles: entry, {}", this); + + final ResolvedTileSpecCollection tiles = sfovTilesAndMatchesForMfov.getResolvedTileSpecs(); + + final Map> modelTiles = new HashMap<>(); + + for (final TileSpec tileSpec : sfovTilesAndMatchesForMfov.getResolvedTileSpecs().getTileSpecs()) { + modelTiles.put(tileSpec.getTileId(), new Tile<>(new TranslationModel2D())); + } + + int connectedPairCount = 0; + for (final CanvasMatches canvasMatches : sfovTilesAndMatchesForMfov.getMatchPairs()) { + // Connect tiles for optimization... + final Tile pModelTile = modelTiles.get(canvasMatches.getpId()); + final Tile qModelTile = modelTiles.get(canvasMatches.getqId()); + + if (pModelTile == null || qModelTile == null) { + LOG.info("alignTiles: skipping pair {} in {}", canvasMatches.toKeyString(), this); + continue; + } + final List pointMatches = CanvasMatchResult.convertMatchesToPointMatchList(canvasMatches.getMatches()); + pModelTile.connect(qModelTile, pointMatches); + connectedPairCount++; + } + + LOG.info("alignTiles: start optimization after connecting {} pairs, {}", + connectedPairCount, this); // Optimize the tiles final TileConfiguration tc = new TileConfiguration(); @@ -382,6 +489,11 @@ private ResolvedTileSpecCollection intensityCorrectTiles( final ImageProcessorCache cache ) { + LOG.info("intensityCorrectTiles: entry with {} tiles and {} match pairs for {}", + tiles.getTileCount(), + tilePairs.size(), + this); + final Map> modelTiles = new HashMap<>(); // Initialize models for each tile spec @@ -424,8 +536,8 @@ private ResolvedTileSpecCollection intensityCorrectTiles( modelTileI.connect(modelTileJ, Collections.singletonList(pointMatch)); } else { - LOG.warn("intensityCorrectTiles: tiles {} and {} do not overlap after alignment", - tileIdI, tileIdJ); + LOG.warn("intensityCorrectTiles: tiles {} and {} do not overlap after alignment of {}", + tileSpecI.toTileBounds(), tileSpecJ.toTileBounds(), this); } } @@ -438,7 +550,7 @@ private ResolvedTileSpecCollection intensityCorrectTiles( final ErrorStatistic errorStatistic = new ErrorStatistic(1001); tc.optimizeSilently(errorStatistic, 0.5, 1000, 100); } catch (final NotEnoughDataPointsException | IllDefinedDataPointsException e) { - throw new RuntimeException("Failed to optimize intensity correction", e); + throw new RuntimeException("Failed to optimize intensity correction for " + this, e); } // Apply the intensity shift to the tiles diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java index e19edddd1..c899250af 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParameters.java @@ -16,11 +16,13 @@ import org.janelia.alignment.util.UrlResourceUtil; import org.janelia.render.client.newsolver.setup.AffineBlockSolverSetup; import org.janelia.render.client.newsolver.setup.IntensityCorrectionSetup; +import org.janelia.render.client.parameter.BeamCorrectionParameters; import org.janelia.render.client.parameter.CreepCorrectionParameters; import org.janelia.render.client.parameter.LayerAsTileParameters; import org.janelia.render.client.parameter.MFOVAsTileParameters; import org.janelia.render.client.parameter.MFOVMontageMatchPatchParameters; import org.janelia.render.client.parameter.MaskHackParameters; +import org.janelia.render.client.parameter.MatchCollectionRenameParameters; import org.janelia.render.client.parameter.MatchCopyParameters; import org.janelia.render.client.parameter.MipmapParameters; import org.janelia.render.client.parameter.MultiProjectParameters; @@ -50,6 +52,8 @@ public class AlignmentPipelineParameters private final UnconnectedCrossMFOVParameters unconnectedCrossMfov; private final TileClusterParameters tileCluster; private final MatchCopyParameters matchCopy; + private final MatchCollectionRenameParameters matchCollectionRename; + private final BeamCorrectionParameters beamCorrection; private final CreepCorrectionParameters creepCorrection; private final AffineBlockSolverSetup affineBlockSolverSetup; private final IntensityCorrectionSetup intensityCorrectionSetup; @@ -79,6 +83,8 @@ public AlignmentPipelineParameters() { null, null, null, + null, + null, null); } @@ -91,6 +97,8 @@ public AlignmentPipelineParameters(final MultiProjectParameters multiProject, final UnconnectedCrossMFOVParameters unconnectedCrossMfov, final TileClusterParameters tileCluster, final MatchCopyParameters matchCopy, + final MatchCollectionRenameParameters matchCollectionRename, + final BeamCorrectionParameters beamCorrection, final CreepCorrectionParameters creepCorrection, final AffineBlockSolverSetup affineBlockSolverSetup, final IntensityCorrectionSetup intensityCorrectionSetup, @@ -109,6 +117,8 @@ public AlignmentPipelineParameters(final MultiProjectParameters multiProject, this.unconnectedCrossMfov = unconnectedCrossMfov; this.tileCluster = tileCluster; this.matchCopy = matchCopy; + this.matchCollectionRename = matchCollectionRename; + this.beamCorrection = beamCorrection; this.creepCorrection = creepCorrection; this.affineBlockSolverSetup = affineBlockSolverSetup; this.intensityCorrectionSetup = intensityCorrectionSetup; @@ -138,6 +148,10 @@ public StackIdNamingGroup getIntensityCorrectedNamingGroup() { return pipelineStackGroups == null ? null : pipelineStackGroups.getIntensityCorrected(); } + public StackIdNamingGroup getOtherNamingGroup() { + return pipelineStackGroups == null ? null : pipelineStackGroups.getOther(); + } + public List getPipelineSteps() { return pipelineSteps; } @@ -166,6 +180,14 @@ public MatchCopyParameters getMatchCopy() { return matchCopy; } + public MatchCollectionRenameParameters getMatchCollectionRename() { + return matchCollectionRename; + } + + public BeamCorrectionParameters getBeamCorrection() { + return beamCorrection; + } + public CreepCorrectionParameters getCreepCorrection() { return creepCorrection; } diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java index 6a5878d34..9dba4fb2e 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineStepId.java @@ -4,24 +4,25 @@ import org.janelia.render.client.spark.MipmapClient; import org.janelia.render.client.spark.ScapeClient; -import org.janelia.render.client.spark.mask.MaskHackClient; +import org.janelia.render.client.spark.tile.MaskHackClient; import org.janelia.render.client.spark.match.ClusterCountClient; import org.janelia.render.client.spark.match.CopyMatchClient; import org.janelia.render.client.spark.match.MultiStagePointMatchClient; +import org.janelia.render.client.spark.multisem.BeamCorrectionSparkClient; import org.janelia.render.client.spark.multisem.CreepCorrectionSparkClient; import org.janelia.render.client.spark.multisem.LayerAsTileClient; -import org.janelia.render.client.spark.multisem.MFOVASTileClient; +import org.janelia.render.client.spark.multisem.MFOVAsTileClient; import org.janelia.render.client.spark.multisem.MFOVMontageMatchPatchClient; +import org.janelia.render.client.spark.multisem.MatchCollectionRenameClient; import org.janelia.render.client.spark.multisem.UnconnectedCrossMFOVClient; import org.janelia.render.client.spark.newsolver.DistributedAffineBlockSolverClient; import org.janelia.render.client.spark.newsolver.DistributedIntensityCorrectionBlockSolverClient; import org.janelia.render.client.spark.tile.RenderTilesClient; +import org.janelia.render.client.spark.tile.TileIdHackClient; import org.janelia.render.client.spark.zspacing.ZPositionCorrectionClient; /** * Identifier for a step in a spark alignment pipeline with a convenience {@link #toStepClient()} builder. - * - * @author Eric Trautman */ public enum AlignmentPipelineStepId { @@ -31,14 +32,17 @@ public enum AlignmentPipelineStepId { FIND_UNCONNECTED_CROSS_MFOVS(UnconnectedCrossMFOVClient::new), FIND_UNCONNECTED_TILES_AND_EDGES(ClusterCountClient::new), FILTER_MATCHES(CopyMatchClient::new), + CORRECT_BEAM_INTENSITY(BeamCorrectionSparkClient::new), CORRECT_CREEP(CreepCorrectionSparkClient::new), ALIGN_TILES(DistributedAffineBlockSolverClient::new), CORRECT_Z_POSITIONS(ZPositionCorrectionClient::new), CORRECT_INTENSITY(DistributedIntensityCorrectionBlockSolverClient::new), HACK_MASK(MaskHackClient::new), + HACK_TILE_ID(TileIdHackClient::new), RENDER_SCAPE_IMAGES(ScapeClient::new), RENDER_TILES(RenderTilesClient::new), - MFOV_AS_TILE(MFOVASTileClient::new), + MFOV_AS_TILE(MFOVAsTileClient::new), + RENAME_MATCH_COLLECTIONS(MatchCollectionRenameClient::new), LAYER_AS_TILE(LayerAsTileClient::new); private final Supplier stepClientSupplier; diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/mask/MaskHackClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/MaskHackClient.java similarity index 99% rename from render-ws-spark-client/src/main/java/org/janelia/render/client/spark/mask/MaskHackClient.java rename to render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/MaskHackClient.java index 8d2df52a4..11a2fdb6d 100644 --- a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/mask/MaskHackClient.java +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/MaskHackClient.java @@ -1,4 +1,4 @@ -package org.janelia.render.client.spark.mask; +package org.janelia.render.client.spark.tile; import com.beust.jcommander.ParametersDelegate; diff --git a/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/TileIdHackClient.java b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/TileIdHackClient.java new file mode 100644 index 000000000..822db1cbf --- /dev/null +++ b/render-ws-spark-client/src/main/java/org/janelia/render/client/spark/tile/TileIdHackClient.java @@ -0,0 +1,163 @@ +package org.janelia.render.client.spark.tile; + +import com.beust.jcommander.ParametersDelegate; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import org.apache.spark.SparkConf; +import org.apache.spark.api.java.JavaRDD; +import org.apache.spark.api.java.JavaSparkContext; +import org.apache.spark.api.java.function.Function; +import org.janelia.alignment.multisem.MultiSemUtilities; +import org.janelia.alignment.spec.ResolvedTileSpecCollection; +import org.janelia.alignment.spec.TileSpec; +import org.janelia.alignment.spec.stack.StackId; +import org.janelia.alignment.spec.stack.StackIdNamingGroup; +import org.janelia.alignment.spec.stack.StackMetaData; +import org.janelia.alignment.spec.stack.StackWithZValues; +import org.janelia.render.client.ClientRunner; +import org.janelia.render.client.RenderDataClient; +import org.janelia.render.client.parameter.CommandLineParameters; +import org.janelia.render.client.parameter.MultiProjectParameters; +import org.janelia.render.client.spark.LogUtilities; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineParameters; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineStep; +import org.janelia.render.client.spark.pipeline.AlignmentPipelineStepId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Spark client for updating the tileId for all tiles in the stack + * to use the new wafer 60 and 61 render order. + */ +public class TileIdHackClient + implements Serializable, AlignmentPipelineStep { + + public static class Parameters extends CommandLineParameters { + @ParametersDelegate + public MultiProjectParameters multiProject; + + public Parameters() { + this(new MultiProjectParameters()); + } + + public Parameters(final MultiProjectParameters multiProject) { + this.multiProject = multiProject; + } + } + + /** Run the client with command line parameters. */ + public static void main(final String[] args) { + final ClientRunner clientRunner = new ClientRunner(args) { + @Override + public void runClient(final String[] args) throws Exception { + final Parameters parameters = new Parameters(); + parameters.parse(args); + final TileIdHackClient client = new TileIdHackClient(); + client.createContextAndRun(parameters); + } + }; + clientRunner.run(); + } + + /** Empty constructor required for alignment pipeline steps. */ + public TileIdHackClient() { + } + + /** Create a spark context and run the client with the specified parameters. */ + public void createContextAndRun(final Parameters clientParameters) throws IOException { + final SparkConf conf = new SparkConf().setAppName(getClass().getSimpleName()); + try (final JavaSparkContext sparkContext = new JavaSparkContext(conf)) { + LOG.info("createContextAndRun: appId is {}", sparkContext.getConf().getAppId()); + copyStackAndFixTileIds(sparkContext, clientParameters); + } + } + + /** Validates the specified pipeline parameters are sufficient. */ + @Override + public void validatePipelineParameters(final AlignmentPipelineParameters pipelineParameters) + throws IllegalArgumentException { + // nothing to validate + } + + /** Run the client as part of an alignment pipeline. */ + public void runPipelineStep(final JavaSparkContext sparkContext, + final AlignmentPipelineParameters pipelineParameters) + throws IllegalArgumentException, IOException { + + final StackIdNamingGroup rawNamingGroup = pipelineParameters.getRawNamingGroup(); + if (rawNamingGroup == null) { + throw new IllegalArgumentException( + "The " + AlignmentPipelineStepId.HACK_TILE_ID + " pipeline step requires that " + + "a 'raw' pipelineStackGroup is defined in the pipeline parameters."); + } + final MultiProjectParameters multiProject = pipelineParameters.getMultiProject(rawNamingGroup); + final Parameters clientParameters = new Parameters(multiProject); + copyStackAndFixTileIds(sparkContext, clientParameters); + } + + @Override + public AlignmentPipelineStepId getDefaultStepId() { + return AlignmentPipelineStepId.HACK_TILE_ID; + } + + private void copyStackAndFixTileIds(final JavaSparkContext sparkContext, + final Parameters clientParameters) + throws IOException { + + LOG.info("copyStackAndFixTileIds: entry, clientParameters={}", clientParameters); + + final MultiProjectParameters multiProjectParameters = clientParameters.multiProject; + final String baseDataUrl = multiProjectParameters.getBaseDataUrl(); + final List stackWithZValuesList = multiProjectParameters.buildListOfStackWithAllZ(); + + final JavaRDD rddStackWithZValues = sparkContext.parallelize(stackWithZValuesList); + + final Function copyAndFixTileIdFunction = stackWithZ -> { + + LogUtilities.setupExecutorLog4j(stackWithZ.toString()); + + final StackId stackId = stackWithZ.getStackId(); + final String stack = stackId.getStack(); + final RenderDataClient sourceDataClient = new RenderDataClient(baseDataUrl, + stackId.getOwner(), + stackId.getProject()); + + sourceDataClient.setStackState(stack, StackMetaData.StackState.LOADING); + + for (final Double z : stackWithZ.getzValues()) { + final ResolvedTileSpecCollection resolvedTiles = sourceDataClient.getResolvedTiles(stack, z); + + // tile specs are keyed by tileId within a collection, so build a new collection after fixing the ids + final List tileSpecsWithFixedIds = new ArrayList<>(resolvedTiles.getTileSpecs()); + tileSpecsWithFixedIds.forEach(tileSpec -> { + final String fixedTileId = + MultiSemUtilities.convertTileIdToUseWafers6061RenderOrder(tileSpec.getTileId()); + tileSpec.setTileId(fixedTileId); + }); + + final ResolvedTileSpecCollection resolvedTilesWithFixedIds = + new ResolvedTileSpecCollection(resolvedTiles.getTransformSpecs(), + tileSpecsWithFixedIds); + + sourceDataClient.deleteStack(stack, z); // remove original tile specs for z layer + sourceDataClient.saveResolvedTiles(resolvedTilesWithFixedIds, stack, z); // save new tile specs for z layer + } + + sourceDataClient.setStackState(stack, StackMetaData.StackState.COMPLETE); + + return null; + }; + + final JavaRDD rddCopyAndFixTileId = rddStackWithZValues.map(copyAndFixTileIdFunction); + rddCopyAndFixTileId.collect(); + + LOG.info("copyStackAndFixTileIds: collected rddCopyAndFixTileId"); + LOG.info("copyStackAndFixTileIds: exit"); + } + + private static final Logger LOG = LoggerFactory.getLogger(TileIdHackClient.class); +} diff --git a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/multisem/MfovPrealignTaskTest.java b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/multisem/MfovPrealignTaskTest.java index 9378bc809..b3b5849c4 100644 --- a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/multisem/MfovPrealignTaskTest.java +++ b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/multisem/MfovPrealignTaskTest.java @@ -6,6 +6,7 @@ import java.util.Date; import java.util.List; +import org.janelia.alignment.match.MatchCollectionId; import org.janelia.alignment.multisem.LayerMFOV; import org.janelia.alignment.spec.stack.StackId; import org.janelia.alignment.spec.stack.StackMetaData; @@ -23,6 +24,37 @@ public class MfovPrealignTaskTest { public static void main(final String[] args) throws Exception { + final String baseDataUrl = "http://renderer-dev.int.janelia.org:8080/render-ws/v1"; + final String owner = "hess_wafers_60_61"; + final String project = "w61_serial_140_to_149"; + final StackId rawSfovStackId = new StackId(owner, project,"w61_s140_r00_gc_icc_test_two_tiles"); + final LayerMFOV z1LayerMfov = new LayerMFOV(1.0, "0023_m0014"); + + // final MatchCollectionId matchCollectionId = null; // set collection to null to derive matches during run + final MatchCollectionId matchCollectionId = new MatchCollectionId(owner, "w61_s140_r00_gc_test_z_1_and_2_match"); + + final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd_HHmmss"); + final String prealignedStackSuffix = "_pa_" + sdf.format(new Date()); + final StackId prealignedStackId = rawSfovStackId.withStackSuffix(prealignedStackSuffix); + + final RenderDataClient dataClient = new RenderDataClient(baseDataUrl, owner, project); + final StackMetaData rawStackMetaData = dataClient.getStackMetaData(rawSfovStackId.getStack()); + dataClient.setupDerivedStack(rawStackMetaData, prealignedStackId.getStack()); + + final MfovPrealignTask z1Task = + new MfovPrealignTask(baseDataUrl, + rawSfovStackId, + prealignedStackId, + z1LayerMfov, + matchCollectionId); + z1Task.run(); + + dataClient.setStackState(prealignedStackId.getStack(), StackMetaData.StackState.COMPLETE); + } + + @SuppressWarnings("unused") + public static void main_2025(final String[] args) throws Exception { + final String mfovRootDirectory = "/nrs/hess/data/hess_wafers_60_61/tiles_mfov"; if (! Files.exists(Paths.get(mfovRootDirectory))) { throw new RuntimeException("The mfovRootDirectory " + mfovRootDirectory + " does not exist " + @@ -49,7 +81,8 @@ public static void main(final String[] args) throws Exception { new MfovPrealignTask(baseDataUrl, rawSfovStackId, prealignedStackId, - z17LayerMfov); + z17LayerMfov, + null); z17Task.run(); final LayerMFOV z18LayerMfov = new LayerMFOV(18.0, "0399_m0035"); @@ -57,7 +90,8 @@ public static void main(final String[] args) throws Exception { new MfovPrealignTask(baseDataUrl, rawSfovStackId, prealignedStackId, - z18LayerMfov); + z18LayerMfov, + null); z18Task.run(); dataClient.setStackState(prealignedStackId.getStack(), StackMetaData.StackState.COMPLETE); @@ -80,8 +114,8 @@ public static void main(final String[] args) throws Exception { final StackId renderedMfovStackId = dynamicMfovStackId.withStackSuffix(mfovAsTile.getRenderedMfovStackSuffix()); final String runTimestamp = new TileRenderParameters().getRunTimestamp(); - final MFOVASTileClient.JavaRenderTilesClientInfoForLayerMfov z17Info = - new MFOVASTileClient.JavaRenderTilesClientInfoForLayerMfov(baseDataUrl, + final MFOVAsTileClient.JavaRenderTilesClientInfoForLayerMfov z17Info = + new MFOVAsTileClient.JavaRenderTilesClientInfoForLayerMfov(baseDataUrl, dynamicMfovStackId, z17LayerMfov, mfovAsTile, @@ -89,8 +123,8 @@ public static void main(final String[] args) throws Exception { z17Info.setupHackStackAndStorage(); z17Info.renderTiles(); - final MFOVASTileClient.JavaRenderTilesClientInfoForLayerMfov z18Info = - new MFOVASTileClient.JavaRenderTilesClientInfoForLayerMfov(baseDataUrl, + final MFOVAsTileClient.JavaRenderTilesClientInfoForLayerMfov z18Info = + new MFOVAsTileClient.JavaRenderTilesClientInfoForLayerMfov(baseDataUrl, dynamicMfovStackId, z18LayerMfov, mfovAsTile, @@ -101,5 +135,4 @@ public static void main(final String[] args) throws Exception { dataClient.setStackState(renderedMfovStackId.getStack(), StackMetaData.StackState.COMPLETE); } - } diff --git a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParametersTest.java b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParametersTest.java index 5ca56fe31..90418e3c6 100644 --- a/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParametersTest.java +++ b/render-ws-spark-client/src/test/java/org/janelia/render/client/spark/pipeline/AlignmentPipelineParametersTest.java @@ -46,6 +46,16 @@ public void testBuildStepClients() AlignmentPipelineStepId.GENERATE_MIPMAPS, stepClient.getDefaultStepId()); } + @Test + public void testNamingGroupSetup() + throws IOException { + final AlignmentPipelineParameters pipelineParameters = loadTestParameters(); + Assert.assertNotNull("rawNamingGroup is null", pipelineParameters.getRawNamingGroup()); + Assert.assertNotNull("alignedNamingGroup is null", pipelineParameters.getAlignedNamingGroup()); + Assert.assertNotNull("intensityCorrectedNamingGroup is null", pipelineParameters.getIntensityCorrectedNamingGroup()); + Assert.assertNotNull("otherNamingGroup is null", pipelineParameters.getOtherNamingGroup()); + } + @Test public void testLoadParametersFromUrl() throws IOException { diff --git a/render-ws-spark-client/src/test/resources/pipeline/msem_alignment_pipeline.json b/render-ws-spark-client/src/test/resources/pipeline/msem_alignment_pipeline.json index e593724c9..9daa56dec 100644 --- a/render-ws-spark-client/src/test/resources/pipeline/msem_alignment_pipeline.json +++ b/render-ws-spark-client/src/test/resources/pipeline/msem_alignment_pipeline.json @@ -17,6 +17,9 @@ }, "intensityCorrected": { "stackPattern": "^c.*_ic$" + }, + "other": { + "stackPattern": "^w61_s..._r.._gc$" } }, "pipelineSteps": [ diff --git a/render-ws-with-mongo-db/db-dump-google-collections.sh b/render-ws-with-mongo-db/db-dump-google-collections.sh index 69c61e718..3d68f96a7 100755 --- a/render-ws-with-mongo-db/db-dump-google-collections.sh +++ b/render-ws-with-mongo-db/db-dump-google-collections.sh @@ -132,7 +132,7 @@ else echo " Select project:" PROJECTS=() - for i in $(seq 0 10 150); do + for i in $(seq 0 10 400); do PROJECTS+=("$(printf "w61_serial_%03d_to_%03d" "$i" "$((i+9))")") done select PROJECT in "${PROJECTS[@]}"; do diff --git a/render-ws-with-mongo-db/list-match-collections.sh b/render-ws-with-mongo-db/list-match-collections.sh index d392029c8..738d3d8a6 100755 --- a/render-ws-with-mongo-db/list-match-collections.sh +++ b/render-ws-with-mongo-db/list-match-collections.sh @@ -2,7 +2,9 @@ OWNER="${1:-hess_wafers_60_61}" -printf "\nMatch collections for owner %s\n" "${OWNER}" +printf "\nMatch collections for owner %s are:\n\n" "${OWNER}" URL="http://localhost:8080/render-ws/v1/owner/${OWNER}/matchCollections" -curl -X GET --silent --header 'Accept: application/json' "${URL}" | jq -r '.[] | "\(.collectionId.name)"' | sort \ No newline at end of file +curl -X GET --silent --header 'Accept: application/json' "${URL}" | jq -r '.[] | "\(.collectionId.name)"' | sort + +echo \ No newline at end of file diff --git a/render-ws-with-mongo-db/other/load-rough-align-data.sh b/render-ws-with-mongo-db/other/load-rough-align-data.sh new file mode 100644 index 000000000..f4067d8d1 --- /dev/null +++ b/render-ws-with-mongo-db/other/load-rough-align-data.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +# Batch identifier appended to each slab group name (edit this for each round of runs). +SLAB_GROUP_SUFFIX="20260811b" + +VM_IPS=(10.150.0.2 10.150.0.3 10.150.0.4 10.150.0.5 10.150.0.6 10.150.0.7) + +printf "\nWhich VM do you want to use?\n\n" +select VM_IP in "${VM_IPS[@]}"; do + if [ -n "${VM_IP}" ]; then + break + else + echo "Invalid selection, try again." + fi +done + +printf "\nWhich wafer do you want to use?\n\n" +select WAFER in 60 61; do + if [ -n "${WAFER}" ]; then + break + else + echo "Invalid selection, try again." + fi +done + +echo +read -rp "Enter the first serial number (a multiple of 5 between 0 and 410): " FIRST_SERIAL_NUMBER + +if [[ ! ${FIRST_SERIAL_NUMBER} =~ ^[0-9]+$ ]]; then + printf "\nExiting, '%s' is not a number\n\n" "${FIRST_SERIAL_NUMBER}" + exit 1 +fi + +# force base 10 so that zero padded values (e.g. 070) are not treated as octal +FIRST_SERIAL_NUMBER=$(( 10#${FIRST_SERIAL_NUMBER} )) + +if (( FIRST_SERIAL_NUMBER > 410 )) || (( FIRST_SERIAL_NUMBER % 5 != 0 )); then + printf "\nExiting, %d is not a multiple of 5 between 0 and 410\n\n" "${FIRST_SERIAL_NUMBER}" + exit 1 +fi + +LAST_SERIAL_NUMBER=$(( FIRST_SERIAL_NUMBER + 4 )) + +# even serial numbers are the first half of a project's slabs, odd ones are the second half +if (( FIRST_SERIAL_NUMBER % 2 == 0 )); then + KEEP_OR_REMOVE="[k]ept" + FIRST_PROJECT_NUMBER=${FIRST_SERIAL_NUMBER} +else + KEEP_OR_REMOVE="[r]emoved" + FIRST_PROJECT_NUMBER=$(( FIRST_SERIAL_NUMBER - 5 )) +fi + +LAST_PROJECT_NUMBER=$(( FIRST_PROJECT_NUMBER + 9 )) + +FIRST_SERIAL=$(printf "%03d" "${FIRST_SERIAL_NUMBER}") +LAST_SERIAL=$(printf "%03d" "${LAST_SERIAL_NUMBER}") +FIRST_PROJECT=$(printf "%03d" "${FIRST_PROJECT_NUMBER}") +LAST_PROJECT=$(printf "%03d" "${LAST_PROJECT_NUMBER}") + +SLAB_GROUP="s${FIRST_SERIAL}_to_s${LAST_SERIAL}_${SLAB_GROUP_SUFFIX}" +BATCH_NAME="rough-w${WAFER}-s${FIRST_SERIAL}-to-s${LAST_SERIAL}" +PROJECT_GROUP="w${WAFER}_serial_${FIRST_PROJECT}_to_${LAST_PROJECT}" + +echo " +Set up for slab group ${SLAB_GROUP} from project group ${PROJECT_GROUP}: + + On ${VM_IP}, run: + + ./db-restore-collections.sh --pattern 'janelia/00_gc/.*s${FIRST_PROJECT}' + + ./other/remove-stacks.sh + + you want stacks to be ${KEEP_OR_REMOVE} + then enter ' 1 2 3 4 5 6 7 8 9 10 ' + + + On launch box, run: + + ./02_run_pipeline.sh ${VM_IP} 00_rough_align/pipe.00.w${WAFER}.icc-match-mat.json 120 4 premium 120 ${BATCH_NAME} + + + After the run completes (typically 8 to 12 hours), on ${VM_IP}, run: + + ./list-stacks.sh + + ./list-match-collections.sh + + ./db-dump-google-collections.sh + + Select database: render + Select stage: 00_par + Select project: ${PROJECT_GROUP} + Enter slab-group: ${SLAB_GROUP} + + Enter collection pattern regex: .* + + Should dump collections to: + /mnt/disks/mongodb_dump_fs/dump/google/00_par/${PROJECT_GROUP}/${SLAB_GROUP}/render + + ./db-dump-google-collections.sh + + Select database: match + Select stage: 00_par + Select project: ${PROJECT_GROUP} + Enter slab-group: ${SLAB_GROUP} + + Enter collection pattern regex: .* + + Should dump collections to: + /mnt/disks/mongodb_dump_fs/dump/google/00_par/${PROJECT_GROUP}/${SLAB_GROUP}/match +"