cleanup mesh constraint

Author: Fabien Servant

meshroom/aliceVision/SfmBootstrapping.py

@@ -1,4 +1,4 @@
-__version__ = "4.1"
+__version__ = "4.2"
 
 from meshroom.core import desc
 from meshroom.core.utils import VERBOSE_LEVEL
@@ -21,8 +21,8 @@ class SfMBootStrapping(desc.AVCommandLineNode):
         desc.ChoiceParam(
             name="method",
             label="Method",
-            description="Bootstrapping method: classic (epipolar geometry), mesh (3D mesh constraints), or depth (depth map information).",
-            values=["classic", "mesh", "depth"],
+            description="Bootstrapping method: classic (epipolar geometry), mesh (3D mesh constraints), mesh_single (mesh without visual parallax), or depth (depth map information).",
+            values=["classic", "mesh", "mesh_single", "depth"],
             value="classic",
         ),
         desc.File(
@@ -36,7 +36,7 @@ class SfMBootStrapping(desc.AVCommandLineNode):
             label="Mesh File",
             description="Mesh file (*.obj).",
             value="",
-            enabled=lambda node: node.method.value == "mesh"
+            enabled=lambda node: node.method.value.startswith("mesh")
         ),
         desc.File(
             name="pairs",

src/aliceVision/sfm/pipeline/expanding/ExpansionChunk.cpp

@@ -159,11 +159,6 @@ bool ExpansionChunk::process(sfmData::SfMData & sfmData, const track::TracksHand
     {
         return false;
     }
-
-    if (_pointFetcherHandler)
-    {
-        setConstraints(sfmData, tracksHandler, validViewIds);
-    }
 
     _historyHandler->saveState(sfmData);
 
@@ -174,18 +169,17 @@ bool ExpansionChunk::process(sfmData::SfMData & sfmData, const track::TracksHand
 
 bool ExpansionChunk::triangulate(sfmData::SfMData & sfmData, const track::TracksHandler & tracksHandler, const std::set<IndexT> & viewIds)
 {
-    ALICEVISION_LOG_INFO("ExpansionChunk::triangulate start");
-    SfmTriangulation triangulation(_triangulationMinPoints, _maxTriangulationError);
-
-    
+    ALICEVISION_LOG_INFO("ExpansionChunk::triangulate start");   
     const bool enableMultiviewTriangulation = true;
 
+    const size_t minPoints = _triangulationHandler->getMinObservations();
+
     if (enableMultiviewTriangulation)
     {
         std::set<IndexT> evaluatedTracks;
         std::map<IndexT, sfmData::Landmark> outputLandmarks;
         std::mt19937 randomNumberGenerator;
-        if (!triangulation.process(sfmData, tracksHandler.getAllTracks(), tracksHandler.getTracksPerView(), 
+        if (!_triangulationHandler->process(sfmData, tracksHandler.getAllTracks(), tracksHandler.getTracksPerView(), 
                                     randomNumberGenerator, viewIds, 
                                     evaluatedTracks, outputLandmarks, false))
         {
@@ -204,7 +198,7 @@ bool ExpansionChunk::triangulate(sfmData::SfMData & sfmData, const track::Tracks
                 landmarks.erase(pl.first);
             }
 
-            if (landmark.getObservations().size() < _triangulationMinPoints)
+            if (landmark.getObservations().size() < minPoints)
             {
                 continue;
             }
@@ -232,7 +226,51 @@ bool ExpansionChunk::triangulate(sfmData::SfMData & sfmData, const track::Tracks
         std::set<IndexT> evaluatedTracks;
         std::map<IndexT, sfmData::Landmark> outputLandmarks;
         std::mt19937 randomNumberGenerator;
-        if (!triangulation.process(sfmData, tracksHandler.getAllTracks(), tracksHandler.getTracksPerView(), 
+        if (!_triangulationHandler->process(sfmData, tracksHandler.getAllTracks(), tracksHandler.getTracksPerView(), 
+                                    randomNumberGenerator, viewIds, 
+                                    evaluatedTracks, outputLandmarks, true))
+        {
+            return false;
+        }
+
+        auto & landmarks = sfmData.getLandmarks();
+        ALICEVISION_LOG_INFO("Existing landmarks : " << landmarks.size());
+
+        for (const auto & pl : outputLandmarks)
+        {
+            const auto & landmark = pl.second;
+
+            if (landmarks.find(pl.first) != landmarks.end())
+            {
+                if (!_ignoreMultiviewOnPrior)
+                {
+                    continue;
+                }
+            }
+
+            if (landmark.getObservations().size() < minPoints)
+            {
+                continue;
+            }
+
+            if (!SfmTriangulation::checkChierality(sfmData, landmark))
+            {
+                continue;
+            }
+
+            landmarks.insert(pl);
+        }
+
+        ALICEVISION_LOG_INFO("New landmarks count : " << landmarks.size());
+        ALICEVISION_LOG_INFO("ExpansionChunk::triangulate end");
+    }
+
+    if (_enableMeshPrior)
+    {
+        std::set<IndexT> evaluatedTracks;
+        std::map<IndexT, sfmData::Landmark> outputLandmarks;
+        std::mt19937 randomNumberGenerator;
+        if (!_triangulationHandler->process(sfmData, tracksHandler.getAllTracks(), tracksHandler.getTracksPerView(), 
                                     randomNumberGenerator, viewIds, 
                                     evaluatedTracks, outputLandmarks, true))
         {
@@ -254,7 +292,7 @@ bool ExpansionChunk::triangulate(sfmData::SfMData & sfmData, const track::Tracks
                 }
             }
 
-            if (landmark.getObservations().size() < _triangulationMinPoints)
+            if (landmark.getObservations().size() < minPoints)
             {
                 continue;
             }
@@ -285,7 +323,7 @@ void ExpansionChunk::addPose(sfmData::SfMData & sfmData, IndexT viewId, const Ei
 
 void ExpansionChunk::setConstraints(sfmData::SfMData & sfmData, const track::TracksHandler & tracksHandler, const std::set<IndexT> & viewIds)
 {
-    ALICEVISION_LOG_INFO("ExpansionChunk::setConstraints start");
+    /*ALICEVISION_LOG_INFO("ExpansionChunk::setConstraints start");
     const track::TracksMap & tracks = tracksHandler.getAllTracks();
     const track::TracksPerView & tracksPerView = tracksHandler.getTracksPerView();
 
@@ -399,7 +437,7 @@ void ExpansionChunk::setConstraints(sfmData::SfMData & sfmData, const track::Tra
     }
 
     ALICEVISION_LOG_INFO("ExpansionChunk::setConstraints added " << constraints.size() << " constraints");
-    ALICEVISION_LOG_INFO("ExpansionChunk::setConstraints end");
+    ALICEVISION_LOG_INFO("ExpansionChunk::setConstraints end");*/
 }
 
 } // namespace sfm

src/aliceVision/sfm/pipeline/expanding/ExpansionChunk.hpp

@@ -11,8 +11,8 @@
 #include <aliceVision/sfmData/SfMData.hpp>
 #include <aliceVision/sfm/pipeline/expanding/ExpansionHistory.hpp>
 #include <aliceVision/sfm/pipeline/expanding/SfmBundle.hpp>
-#include <aliceVision/sfm/pipeline/expanding/PointFetcher.hpp>
 #include <aliceVision/sfm/pipeline/expanding/SfmResection.hpp>
+#include <aliceVision/sfm/pipeline/expanding/SfmTriangulation.hpp>
 
 namespace aliceVision {
 namespace sfm {
@@ -54,16 +54,7 @@ class ExpansionChunk
     }
 
     /**
-     * brief setup the point fetcher handler
-     * @param pointFetcher a unique ptr. the Ownership will be taken
-    */
-    void setPointFetcherHandler(PointFetcher::uptr & pointFetcherHandler)
-    {
-        _pointFetcherHandler = std::move(pointFetcherHandler);
-    }
-
-    /**
-     * brief setup the point fetcher handler
+     * brief setup the Resection handler
      * @param resectionHandler a unique ptr. the Ownership will be taken
     */
     void setResectionHandler(SfmResection::uptr & resectionHandler)
@@ -72,21 +63,12 @@ class ExpansionChunk
     }
 
     /**
-     * @brief set the minimal number of points to enable triangulation of a track
-     * @param count the number of points
-    */
-    void setTriangulationMinPoints(size_t count)
-    {
-        _triangulationMinPoints = count;
-    }
-
-    /**
-     * @brief set the maximal reprojection error in the triangulation process.
-     * @param count the number of points
+     * brief setup the Triangulation handler
+     * @param triangulationHandler a unique ptr. the Ownership will be taken
     */
-    void setTriangulationMaxError(double error)
+    void setTriangulationHandler(SfmTriangulation::uptr & triangulationHandler)
     {
-        _maxTriangulationError = error;
+        _triangulationHandler = std::move(triangulationHandler);
     }
 
     /**
@@ -159,16 +141,15 @@ class ExpansionChunk
 private:
     SfmBundle::uptr _bundleHandler;
     ExpansionHistory::sptr _historyHandler;
-    PointFetcher::uptr _pointFetcherHandler;
     std::set<IndexT> _ignoredViews;
     SfmResection::uptr _resectionHandler;
+    SfmTriangulation::uptr _triangulationHandler;
 
 private:    
-    size_t _triangulationMinPoints = 2;
     double _minTriangulationAngleDegrees = 3.0;
-    double _maxTriangulationError = 8.0;
     size_t _weakResectionSize = 100;
     bool _enableDepthPrior = true;
+    bool _enableMeshPrior = true;
     bool _ignoreMultiviewOnPrior = false;
 };
 

src/aliceVision/sfm/pipeline/expanding/PointFetcher.hpp

@@ -18,6 +18,7 @@ class PointFetcher
 {
 public:
     using uptr = std::unique_ptr<PointFetcher>;
+    using sptr = std::shared_ptr<PointFetcher>;
 
 public:
     /**

src/aliceVision/sfm/pipeline/expanding/SfmBundle.hpp

@@ -106,7 +106,7 @@ class SfmBundle
     double _maxReprojectionError = 4.0;
     double _minAngleForLandmark = 2.0;
     double _maxConstraintDistance = 1.0;
-    size_t _minTrackLength = 2;
+    size_t _minTrackLength = 1;
     size_t _minPointsPerPose = 30;
     size_t _bundleAdjustmentMaxOutlier = 50;
     size_t _minNbCamerasToRefinePrincipalPoint = 3;

src/aliceVision/sfm/pipeline/expanding/SfmTriangulation.cpp

@@ -86,9 +86,19 @@ bool SfmTriangulation::process(
         sfmData::Landmark result;
         if (useDepthPrior)
         {
-            if (!processTrackWithPrior(sfmData, track, randomNumberGenerator, trackViewsFiltered, result))
+            if (_pointFetcherHandler)
             {
-                continue;
+                if (!processTrackWithPointFetcher(sfmData, track, randomNumberGenerator, trackViewsFiltered, result))
+                {
+                    continue;
+                }
+            }
+            else 
+            {
+                if (!processTrackWithPrior(sfmData, track, randomNumberGenerator, trackViewsFiltered, result))
+                {
+                    continue;
+                }
             }
         }
         else 
@@ -284,6 +294,108 @@ bool SfmTriangulation::processTrackWithPrior(
     return true;
 }
 
+bool SfmTriangulation::processTrackWithPointFetcher(
+            const sfmData::SfMData & sfmData,
+            const track::Track & track,
+            std::mt19937 &randomNumberGenerator,
+            const std::set<IndexT> & viewIds,
+            sfmData::Landmark & result
+        )
+{
+    size_t bestInliersCount = 0;
+    std::vector<std::pair<Vec3, Vec3>> possibleParameters;
+
+    //For each observed view in the track
+    for (auto referenceViewId : viewIds)
+    {   
+        if (track.featPerView.find(referenceViewId) == track.featPerView.end())
+        {
+            continue;
+        }
+
+        //Look if this observation has an associated depth
+        const auto & refTrackItem = track.featPerView.at(referenceViewId);
+
+        const sfmData::View & v = sfmData.getView(referenceViewId);
+        const sfmData::CameraPose & cp = sfmData.getAbsolutePose(v.getPoseId());
+        const camera::IntrinsicBase & intrinsics = *sfmData.getIntrinsics().at(v.getIntrinsicId());
+
+        _pointFetcherHandler->setPose(cp.getTransform());
+
+        Vec3 point, normal;
+        if (!_pointFetcherHandler->pickPointAndNormal(point, normal, intrinsics, refTrackItem.coords))
+        {
+            continue;
+        }
+
+        possibleParameters.push_back(std::make_pair(point, normal));
+    }
+
+
+    
+    //Consider each point
+    for (int idRef = 0; idRef < possibleParameters.size(); idRef++)
+    {
+        const Vec3 & refpt = possibleParameters[idRef].first;
+        const Vec3 & refnormal = possibleParameters[idRef].second;
+
+        //Make sure this point is not dependent on parallax 
+        //As it does not need parallax to estimate its depth
+        sfmData::Landmark landmark;
+        landmark.setParallaxRobust(true);
+        landmark.X = refpt;
+        landmark.descType = track.descType;
+
+        //For each observed view in the track
+        for (auto viewId: viewIds)
+        {   
+            if (track.featPerView.find(viewId) == track.featPerView.end())
+            {
+                continue;
+            }
+
+            const auto & trackItem = track.featPerView.at(viewId);
+
+            const sfmData::View & view = sfmData.getView(viewId);
+            const camera::IntrinsicBase & intrinsic = sfmData.getIntrinsic(view.getIntrinsicId());
+            const geometry::Pose3 pose = sfmData.getPose(view).getTransform();    
+
+            const Vec2 est = intrinsic.transformProject(pose, refpt.homogeneous(), true);
+            const Vec2 mes = trackItem.coords.cast<double>();
+            double err = (est - mes).norm() / trackItem.scale;
+
+            //Use _maxError as threshold for inliers/outlier detection
+            if (err > _maxError)
+            {
+                continue;
+            }
+
+            //Create and associated an observation to the landmark
+            sfmData::Observation & o = landmark.getObservations()[viewId];
+            o.setFeatureId(trackItem.featureId);
+            o.setScale(trackItem.scale);
+            o.setCoordinates(trackItem.coords);
+            o.setDepth(trackItem.depth);
+        }
+
+        //Store the landmark if it's better than the previously estimated one
+        int count = landmark.getObservations().size();
+        if (count > bestInliersCount)
+        {
+            bestInliersCount = count;
+            result = landmark;
+        }
+    }
+
+    //One inlier is the reference, so we need at least 2 inliers
+    if (bestInliersCount < 2)
+    {
+        return false;
+    }
+
+    return true;
+}
+
 bool SfmTriangulation::checkChierality(const sfmData::SfMData & sfmData, const sfmData::Landmark & landmark)
 {
     for (const auto & pRefObs : landmark.getObservations())