Merge pull request #628 from alicevision/dev_matchingIO

FeatureMatching: Prefix output matches files when using "ranges" to avoid overwrites

src/aliceVision/matching/indMatch_test.cpp

@@ -76,12 +76,32 @@ BOOST_AUTO_TEST_CASE(IndMatch_IO)
     matches[std::make_pair(1,2)][EImageDescriberType::UNKNOWN] = {{0,0},{1,1}, {2,2}};
 
     BOOST_CHECK(Save(matches, testFolder, "txt", true));
+    // Create an additional pair-wise matches entry
+    matches[std::make_pair(0,2)][EImageDescriberType::UNKNOWN] = {{3,3},{4,4}};
+
+    // Don't clear matches and reload saved file
     BOOST_CHECK(Load(matches, viewsKeys, {testFolder}, {EImageDescriberType::UNKNOWN}));
-    BOOST_CHECK_EQUAL(2, matches.size());
+    BOOST_CHECK_EQUAL(3, matches.size());
     BOOST_CHECK_EQUAL(1, matches.count(std::make_pair(0,1)));
     BOOST_CHECK_EQUAL(1, matches.count(std::make_pair(1,2)));
+    BOOST_CHECK_EQUAL(1, matches.count(std::make_pair(0,2)));
+    // 'matches' was not cleared, pair-wise matches are accumulated ({0, 1} and {1,2} contains duplicates)
+    BOOST_CHECK_EQUAL(4, matches.at(std::make_pair(0,1)).at(EImageDescriberType::UNKNOWN).size());
+    BOOST_CHECK_EQUAL(6, matches.at(std::make_pair(1,2)).at(EImageDescriberType::UNKNOWN).size());
+    BOOST_CHECK_EQUAL(2, matches.at(std::make_pair(0,2)).at(EImageDescriberType::UNKNOWN).size());
+
+    // Deduplicate matches
+    //  - {0, 1} has duplicates
+    BOOST_CHECK(IndMatch::getDeduplicated(matches.at(std::make_pair(0,1)).at(EImageDescriberType::UNKNOWN)));
+    //  - {1, 2} has duplicates
+    BOOST_CHECK(IndMatch::getDeduplicated(matches.at(std::make_pair(1,2)).at(EImageDescriberType::UNKNOWN)));
+    //  - {0, 2} does not have duplicates
+    BOOST_CHECK(!IndMatch::getDeduplicated(matches.at(std::make_pair(0,2)).at(EImageDescriberType::UNKNOWN)));
+
+    // Check matches count after deduplication
     BOOST_CHECK_EQUAL(2, matches.at(std::make_pair(0,1)).at(EImageDescriberType::UNKNOWN).size());
     BOOST_CHECK_EQUAL(3, matches.at(std::make_pair(1,2)).at(EImageDescriberType::UNKNOWN).size());
+    BOOST_CHECK_EQUAL(2, matches.at(std::make_pair(0,2)).at(EImageDescriberType::UNKNOWN).size());
     fs::remove_all("./4/");
   }
   boost::filesystem::remove_all(testFolder);

src/aliceVision/matching/io.cpp

@@ -11,6 +11,7 @@
 #include <aliceVision/system/Logger.hpp>
 
 #include <boost/filesystem.hpp>
+#include <boost/range/iterator_range.hpp>
 
 #include <map>
 #include <fstream>
@@ -133,7 +134,7 @@ void filterMatchesByDesc(
 }
 
 
-bool LoadMatchFilePerImage(
+std::size_t LoadMatchFilePerImage(
   PairwiseMatches& matches,
   const std::set<IndexT>& viewsKeys,
   const std::string& folder,
@@ -167,21 +168,63 @@ bool LoadMatchFilePerImage(
       }
     }
   }
-  if(nbLoadedMatchFiles == 0)
+  return nbLoadedMatchFiles;
+}
+
+/**
+ * Load and add pair-wise matches to \p matches from all files in \p folder matching \p pattern.
+ * @param[out] matches PairwiseMatches to add loaded matches to
+ * @param[in] folder Folder to load matches files from
+ * @param[in] pattern Pattern that files must respect to be loaded
+ */
+std::size_t loadMatchesFromFolder(PairwiseMatches& matches, const std::string& folder, const std::string& pattern)
+{
+  std::size_t nbLoadedMatchFiles = 0;
+  std::vector<std::string> matchFiles;
+  // list all matches files in 'folder' matching (i.e containing) 'pattern'
+  for(const auto& entry : boost::make_iterator_range(fs::directory_iterator(folder), {}))
   {
-    ALICEVISION_LOG_WARNING("No matches file loaded in: " << folder);
-    return false;
+    if(entry.path().string().find(pattern) != std::string::npos)
+    {
+      matchFiles.push_back(entry.path().string());
+    }
   }
-  ALICEVISION_LOG_TRACE("Matches per image pair");
-  for(const auto& imagePairIt: matches)
+
+  #pragma omp parallel for num_threads(3)
+  for(int i = 0; i < matchFiles.size(); ++i)
   {
-    std::stringstream ss;
-    ss << " * " << imagePairIt.first.first << "-" << imagePairIt.first.second << ": " << imagePairIt.second.getNbAllMatches() << "    ";
-    for(const auto& matchesPerDeskIt: imagePairIt.second)
-       ss << " [" << feature::EImageDescriberType_enumToString(matchesPerDeskIt.first) << ": " << matchesPerDeskIt.second.size() << "]";
-    ALICEVISION_LOG_TRACE(ss.str());
+    const std::string& matchFile = matchFiles[i];
+    PairwiseMatches fileMatches;
+    ALICEVISION_LOG_DEBUG("Loading match file: " << matchFile);
+    if(!LoadMatchFile(fileMatches, matchFile))
+    {
+      ALICEVISION_LOG_WARNING("Unable to load match file: " << matchFile);
+      continue;
+    }
+    #pragma omp critical
+    {
+    for(const auto& matchesPerView: fileMatches)
+    {
+      const Pair& pair = matchesPerView.first;
+      const MatchesPerDescType& pairMatches = matchesPerView.second;
+      for(const auto& matchesPerDescType : pairMatches)
+      {
+        const feature::EImageDescriberType& descType = matchesPerDescType.first;
+        const auto& pairMatches = matchesPerDescType.second;
+        // merge in global map
+        std::copy(
+          std::make_move_iterator(pairMatches.begin()), 
+          std::make_move_iterator(pairMatches.end()), 
+          std::back_inserter(matches[pair][descType])
+        );
+      }
+    }
+    ++nbLoadedMatchFiles;
+    }   
   }
-  return true;
+  if(!nbLoadedMatchFiles)
+    ALICEVISION_LOG_WARNING("No matches file loaded in: " << folder);
+  return nbLoadedMatchFiles;
 }
 
 bool Load(
@@ -191,22 +234,37 @@ bool Load(
   const std::vector<feature::EImageDescriberType>& descTypesFilter,
   const int maxNbMatches)
 {
-  bool res = false;
-  const std::string fileName = "matches.txt";
+  std::size_t nbLoadedMatchFiles = 0;
+  const std::string pattern = "matches.txt";
 
-  for(const std::string& folder : folders)
-  {
-    const fs::path filePath = fs::path(folder) / fileName;
+  // build up a set with normalized paths to remove duplicates
+  std::set<std::string> foldersSet;
+  for(const auto& folder : folders)
+    foldersSet.insert(fs::canonical(folder).string());
 
-    if(fs::exists(filePath))
-      res = LoadMatchFile(matches, filePath.string());
-    else
-      res = LoadMatchFilePerImage(matches, viewsKeysFilter, folder, fileName);
+  for(const auto& folder : foldersSet)
+  {
+    nbLoadedMatchFiles += loadMatchesFromFolder(matches, folder, pattern);
   }
 
-  if(!res)
+  if(!nbLoadedMatchFiles)
     return false;
 
+  const auto logMatches = [](const PairwiseMatches& m)
+  {
+    for(const auto& imagePairIt: m)
+    {
+      std::stringstream ss;
+      ss << " * " << imagePairIt.first.first << "-" << imagePairIt.first.second << ": " << imagePairIt.second.getNbAllMatches() << "    ";
+      for(const auto& matchesPerDeskIt: imagePairIt.second)
+         ss << " [" << feature::EImageDescriberType_enumToString(matchesPerDeskIt.first) << ": " << matchesPerDeskIt.second.size() << "]";
+      ALICEVISION_LOG_TRACE(ss.str());
+    }
+  };
+
+  ALICEVISION_LOG_TRACE("Matches per image pair (before filtering):");
+  logMatches(matches);
+
   if(!viewsKeysFilter.empty())
     filterMatchesByViews(matches, viewsKeysFilter);
 
@@ -216,19 +274,10 @@ bool Load(
   if(maxNbMatches > 0)
     filterTopMatches(matches, maxNbMatches);
 
-  ALICEVISION_LOG_TRACE("Matches per image pair");
-  for(const auto& imagePairIt: matches)
-  {
-    std::stringstream ss;
-    ss << " * " << imagePairIt.first.first << "-" << imagePairIt.first.second << ": " << imagePairIt.second.getNbAllMatches() << "    ";
-    for(const auto& matchesPerDeskIt: imagePairIt.second)
-    {
-       ss << " [" << feature::EImageDescriberType_enumToString(matchesPerDeskIt.first) << ": " << matchesPerDeskIt.second.size() << "]";
-    }
-    ALICEVISION_LOG_TRACE(ss.str());
-  }
+  ALICEVISION_LOG_TRACE("Matches per image pair (after filtering):");
+  logMatches(matches);
 
-  return res;
+  return true;
 }
 
 
@@ -331,9 +380,11 @@ bool Save(
   const PairwiseMatches & matches,
   const std::string & folder,
   const std::string & extension,
-  bool matchFilePerImage)
+  bool matchFilePerImage,
+  const std::string& prefix
+  )
 {
-  const std::string filename = "matches." + extension;
+  const std::string filename = prefix + "matches." + extension;
   MatchExporter exporter(matches, folder, filename);
 
   if(matchFilePerImage)

src/aliceVision/matching/io.hpp

@@ -75,12 +75,14 @@ void filterTopMatches(
  * @param[in] extension: txt or bin file format
  * @param[in] matchFilePerImage: do we store a global match file
  *            or one match file per image
+ * @param[in] prefix: optional prefix for the output file(s)
  */
 bool Save(
   const PairwiseMatches& matches,
   const std::string& folder,
   const std::string& extension,
-  bool matchFilePerImage);
+  bool matchFilePerImage,
+  const std::string& prefix="");
 
 }  // namespace matching
 }  // namespace aliceVision

src/aliceVision/sfm/pipeline/regionsIO.cpp

@@ -84,7 +84,12 @@ std::unique_ptr<feature::Regions> loadFeatures(const std::vector<std::string>& f
 
   std::string featFilename;
 
-  for(const std::string& folder : folders)
+  // build up a set with normalized paths to remove duplicates
+  std::set<std::string> foldersSet;
+  for(const auto& folder : folders)
+    foldersSet.insert(fs::canonical(folder).string());
+
+  for(const auto& folder : foldersSet)
   {
     const fs::path featPath = fs::path(folder) / std::string(basename + "." + imageDescriberTypeName + ".feat");
     if(fs::exists(featPath))

src/aliceVision/sfmData/CMakeLists.txt

@@ -30,3 +30,9 @@ alicevision_add_library(aliceVision_sfmData
 )
 
 # Unit tests
+
+alicevision_add_test(sfmData_test.cpp
+  NAME "sfmData"
+  LINKS aliceVision_sfmData
+        aliceVision_system
+)

src/aliceVision/sfmData/SfMData.cpp

@@ -89,22 +89,79 @@ bool SfMData::operator==(const SfMData& other) const {
 
 }
 
-std::vector<std::string> SfMData::getFeaturesFolders() const
+/**
+ * @brief Convert paths in \p folders to absolute paths using \p absolutePath parent folder as base.
+ * @param[in] folders list of paths to convert
+ * @param[in] absolutePath filepath which parent folder should be used as base for absolute path conversion
+ * @return the list of converted absolute paths or input folder if absolutePath is empty
+ */
+std::vector<std::string> toAbsoluteFolders(const std::vector<std::string>& folders, const std::string& absolutePath)
 {
-  fs::path sfmFolder = fs::path(_absolutePath).parent_path();
-  std::vector<std::string> absolutePaths(_featuresFolders.size());
+  // if absolute path is not set, return input folders
+  if(absolutePath.empty())
+    return folders;
+  // else, convert relative paths to absolute paths
+  std::vector<std::string> absolutePaths(folders.size());
   for(int i = 0; i < absolutePaths.size(); ++i)
-    absolutePaths.at(i) = fs::canonical(fs::path(_featuresFolders.at(i)), sfmFolder).string();
+    absolutePaths.at(i) = fs::canonical(folders.at(i), fs::path(absolutePath).parent_path()).string();
   return absolutePaths;
 }
 
+/** 
+ * @brief Add paths contained in \p folders to \p dst as relative paths to \p absolutePath.
+ *        Paths already present in \p dst are omitted.
+ * @param[in] dst list in which paths should be added
+ * @param[in] folders paths to add to \p dst as relative folders
+ * @param[in] absolutePath filepath which parent folder should be used as base for relative path conversions
+ */
+void addAsRelativeFolders(std::vector<std::string>& dst, const std::vector<std::string>& folders, const std::string& absolutePath)
+{
+  for(auto folderPath: folders) 
+  {
+    // if absolutePath is set, convert to relative path
+    if(!absolutePath.empty() && fs::path(folderPath).is_absolute())
+    {
+      folderPath = fs::relative(folderPath, fs::path(absolutePath).parent_path()).string();
+    }
+    // add path only if not already in dst
+    if(std::find(dst.begin(), dst.end(), folderPath) == dst.end())
+    {
+      dst.emplace_back(folderPath);
+    }
+  }
+}
+
+std::vector<std::string> SfMData::getFeaturesFolders() const
+{
+  return toAbsoluteFolders(_featuresFolders, _absolutePath);
+}
+
 std::vector<std::string> SfMData::getMatchesFolders() const
 {
-  fs::path sfmFolder = fs::path(_absolutePath).parent_path();
-  std::vector<std::string> absolutePaths(_matchesFolders.size());
-  for(int i = 0; i < absolutePaths.size(); ++i)
-    absolutePaths.at(i) = fs::canonical(fs::path(_matchesFolders.at(i)), sfmFolder).string();
-  return absolutePaths;
+  return toAbsoluteFolders(_matchesFolders, _absolutePath);
+}
+
+void SfMData::addFeaturesFolders(const std::vector<std::string>& folders)
+{
+  addAsRelativeFolders(_featuresFolders, folders, _absolutePath);
+}
+
+void SfMData::addMatchesFolders(const std::vector<std::string>& folders)
+{
+  addAsRelativeFolders(_matchesFolders, folders, _absolutePath);
+}
+
+void SfMData::setAbsolutePath(const std::string& path)
+{
+  // get absolute path to features/matches folders
+  const std::vector<std::string> featuresFolders = getFeaturesFolders();
+  const std::vector<std::string> matchesFolders = getMatchesFolders();
+  // change internal absolute path
+  _absolutePath = path;
+  // re-set features/matches folders
+  // they will be converted back to relative paths based on updated _absolutePath
+  setFeaturesFolders(featuresFolders);
+  setMatchesFolders(matchesFolders);
 }
 
 std::set<IndexT> SfMData::getValidViews() const
@@ -169,10 +226,10 @@ void SfMData::combine(const SfMData& sfmData)
     throw std::runtime_error("Can't combine two SfMData with rigs");
 
   // feature folder
-  _featuresFolders.insert(_featuresFolders.end(), sfmData._featuresFolders.begin(), sfmData._featuresFolders.end());
+  addFeaturesFolders(sfmData.getFeaturesFolders());
 
   // matching folder
-  _matchesFolders.insert(_matchesFolders.end(), sfmData._matchesFolders.begin(), sfmData._matchesFolders.end());
+  addMatchesFolders(sfmData.getMatchesFolders());
 
   // views
   views.insert(sfmData.views.begin(), sfmData.views.end());