// Copyright (c) 2022, ETH Zurich and UNC Chapel Hill. // All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // // * Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. // // * Neither the name of ETH Zurich and UNC Chapel Hill nor the names of // its contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE // ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE // LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR // CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF // SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN // CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) // ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE // POSSIBILITY OF SUCH DAMAGE. // // Author: Johannes L. Schoenberger (jsch-at-demuc-dot-de) #include "exe/sfm.h" #include #include #include "base/reconstruction.h" #include "controllers/automatic_reconstruction.h" #include "controllers/bundle_adjustment.h" #include "controllers/hierarchical_mapper.h" #include "exe/gui.h" #include "util/misc.h" #include "util/opengl_utils.h" #include "util/option_manager.h" namespace colmap { int RunAutomaticReconstructor(int argc, char** argv) { AutomaticReconstructionController::Options reconstruction_options; std::string data_type = "individual"; std::string quality = "high"; std::string mesher = "poisson"; OptionManager options; options.AddRequiredOption("workspace_path", &reconstruction_options.workspace_path); options.AddRequiredOption("image_path", &reconstruction_options.image_path); options.AddDefaultOption("mask_path", &reconstruction_options.mask_path); options.AddDefaultOption("vocab_tree_path", &reconstruction_options.vocab_tree_path); options.AddDefaultOption("data_type", &data_type, "{individual, video, internet}"); options.AddDefaultOption("quality", &quality, "{low, medium, high, extreme}"); options.AddDefaultOption("camera_model", &reconstruction_options.camera_model); options.AddDefaultOption("single_camera", &reconstruction_options.single_camera); options.AddDefaultOption("sparse", &reconstruction_options.sparse); options.AddDefaultOption("dense", &reconstruction_options.dense); options.AddDefaultOption("mesher", &mesher, "{poisson, delaunay}"); options.AddDefaultOption("num_threads", &reconstruction_options.num_threads); options.AddDefaultOption("use_gpu", &reconstruction_options.use_gpu); options.AddDefaultOption("gpu_index", &reconstruction_options.gpu_index); options.Parse(argc, argv); StringToLower(&data_type); if (data_type == "individual") { reconstruction_options.data_type = AutomaticReconstructionController::DataType::INDIVIDUAL; } else if (data_type == "video") { reconstruction_options.data_type = AutomaticReconstructionController::DataType::VIDEO; } else if (data_type == "internet") { reconstruction_options.data_type = AutomaticReconstructionController::DataType::INTERNET; } else { LOG(FATAL) << "Invalid data type provided"; } StringToLower(&quality); if (quality == "low") { reconstruction_options.quality = AutomaticReconstructionController::Quality::LOW; } else if (quality == "medium") { reconstruction_options.quality = AutomaticReconstructionController::Quality::MEDIUM; } else if (quality == "high") { reconstruction_options.quality = AutomaticReconstructionController::Quality::HIGH; } else if (quality == "extreme") { reconstruction_options.quality = AutomaticReconstructionController::Quality::EXTREME; } else { LOG(FATAL) << "Invalid quality provided"; } StringToLower(&mesher); if (mesher == "poisson") { reconstruction_options.mesher = AutomaticReconstructionController::Mesher::POISSON; } else if (mesher == "delaunay") { reconstruction_options.mesher = AutomaticReconstructionController::Mesher::DELAUNAY; } else { LOG(FATAL) << "Invalid mesher provided"; } ReconstructionManager reconstruction_manager; if (reconstruction_options.use_gpu && kUseOpenGL) { QApplication app(argc, argv); AutomaticReconstructionController controller(reconstruction_options, &reconstruction_manager); RunThreadWithOpenGLContext(&controller); } else { AutomaticReconstructionController controller(reconstruction_options, &reconstruction_manager); controller.Start(); controller.Wait(); } return EXIT_SUCCESS; } int RunBundleAdjuster(int argc, char** argv) { std::string input_path; std::string output_path; OptionManager options; options.AddRequiredOption("input_path", &input_path); options.AddRequiredOption("output_path", &output_path); options.AddBundleAdjustmentOptions(); options.Parse(argc, argv); if (!ExistsDir(input_path)) { std::cerr << "ERROR: `input_path` is not a directory" << std::endl; return EXIT_FAILURE; } if (!ExistsDir(output_path)) { std::cerr << "ERROR: `output_path` is not a directory" << std::endl; return EXIT_FAILURE; } Reconstruction reconstruction; reconstruction.Read(input_path); BundleAdjustmentController ba_controller(options, &reconstruction); ba_controller.Start(); ba_controller.Wait(); reconstruction.Write(output_path); return EXIT_SUCCESS; } int RunColorExtractor(int argc, char** argv) { std::string input_path; std::string output_path; OptionManager options; options.AddImageOptions(); options.AddDefaultOption("input_path", &input_path); options.AddRequiredOption("output_path", &output_path); options.Parse(argc, argv); Reconstruction reconstruction; reconstruction.Read(input_path); reconstruction.ExtractColorsForAllImages(*options.image_path); reconstruction.Write(output_path); return EXIT_SUCCESS; } int RunMapper(int argc, char** argv) { std::string input_path; std::string output_path; std::string image_list_path; OptionManager options; options.AddDatabaseOptions(); options.AddImageOptions(); options.AddDefaultOption("input_path", &input_path); options.AddRequiredOption("output_path", &output_path); options.AddDefaultOption("image_list_path", &image_list_path); options.AddMapperOptions(); options.Parse(argc, argv); if (!ExistsDir(output_path)) { std::cerr << "ERROR: `output_path` is not a directory." << std::endl; return EXIT_FAILURE; } if (!image_list_path.empty()) { const auto image_names = ReadTextFileLines(image_list_path); options.mapper->image_names = std::unordered_set(image_names.begin(), image_names.end()); } ReconstructionManager reconstruction_manager; if (input_path != "") { if (!ExistsDir(input_path)) { std::cerr << "ERROR: `input_path` is not a directory." << std::endl; return EXIT_FAILURE; } reconstruction_manager.Read(input_path); } IncrementalMapperController mapper(options.mapper.get(), *options.image_path, *options.database_path, &reconstruction_manager); // In case a new reconstruction is started, write results of individual sub- // models to as their reconstruction finishes instead of writing all results // after all reconstructions finished. size_t prev_num_reconstructions = 0; if (input_path == "") { mapper.AddCallback( IncrementalMapperController::LAST_IMAGE_REG_CALLBACK, [&]() { // If the number of reconstructions has not changed, the last model // was discarded for some reason. if (reconstruction_manager.Size() > prev_num_reconstructions) { const std::string reconstruction_path = JoinPaths( output_path, std::to_string(prev_num_reconstructions)); const auto& reconstruction = reconstruction_manager.Get(prev_num_reconstructions); CreateDirIfNotExists(reconstruction_path); reconstruction.Write(reconstruction_path); options.Write(JoinPaths(reconstruction_path, "project.ini")); prev_num_reconstructions = reconstruction_manager.Size(); } }); } mapper.Start(); mapper.Wait(); if (reconstruction_manager.Size() == 0) { std::cerr << "ERROR: failed to create sparse model" << std::endl; return EXIT_FAILURE; } // In case the reconstruction is continued from an existing reconstruction, do // not create sub-folders but directly write the results. if (input_path != "" && reconstruction_manager.Size() > 0) { reconstruction_manager.Get(0).Write(output_path); } return EXIT_SUCCESS; } int RunHierarchicalMapper(int argc, char** argv) { HierarchicalMapperController::Options hierarchical_options; SceneClustering::Options clustering_options; std::string output_path; OptionManager options; options.AddRequiredOption("database_path", &hierarchical_options.database_path); options.AddRequiredOption("image_path", &hierarchical_options.image_path); options.AddRequiredOption("output_path", &output_path); options.AddDefaultOption("num_workers", &hierarchical_options.num_workers); options.AddDefaultOption("image_overlap", &clustering_options.image_overlap); options.AddDefaultOption("leaf_max_num_images", &clustering_options.leaf_max_num_images); options.AddMapperOptions(); options.Parse(argc, argv); if (!ExistsDir(output_path)) { std::cerr << "ERROR: `output_path` is not a directory." << std::endl; return EXIT_FAILURE; } ReconstructionManager reconstruction_manager; HierarchicalMapperController hierarchical_mapper( hierarchical_options, clustering_options, *options.mapper, &reconstruction_manager); hierarchical_mapper.Start(); hierarchical_mapper.Wait(); if (reconstruction_manager.Size() == 0) { std::cerr << "ERROR: failed to create sparse model" << std::endl; return EXIT_FAILURE; } reconstruction_manager.Write(output_path, &options); return EXIT_SUCCESS; } int RunPointFiltering(int argc, char** argv) { std::string input_path; std::string output_path; size_t min_track_len = 2; double max_reproj_error = 4.0; double min_tri_angle = 1.5; OptionManager options; options.AddRequiredOption("input_path", &input_path); options.AddRequiredOption("output_path", &output_path); options.AddDefaultOption("min_track_len", &min_track_len); options.AddDefaultOption("max_reproj_error", &max_reproj_error); options.AddDefaultOption("min_tri_angle", &min_tri_angle); options.Parse(argc, argv); Reconstruction reconstruction; reconstruction.Read(input_path); size_t num_filtered = reconstruction.FilterAllPoints3D(max_reproj_error, min_tri_angle); for (const auto point3D_id : reconstruction.Point3DIds()) { const auto& point3D = reconstruction.Point3D(point3D_id); if (point3D.Track().Length() < min_track_len) { num_filtered += point3D.Track().Length(); reconstruction.DeletePoint3D(point3D_id); } } std::cout << "Filtered observations: " << num_filtered << std::endl; reconstruction.Write(output_path); return EXIT_SUCCESS; } int RunPointTriangulator(int argc, char** argv) { std::string input_path; std::string output_path; bool clear_points = false; OptionManager options; options.AddDatabaseOptions(); options.AddImageOptions(); options.AddRequiredOption("input_path", &input_path); options.AddRequiredOption("output_path", &output_path); options.AddDefaultOption( "clear_points", &clear_points, "Whether to clear all existing points and observations"); options.AddMapperOptions(); options.Parse(argc, argv); if (!ExistsDir(input_path)) { std::cerr << "ERROR: `input_path` is not a directory" << std::endl; return EXIT_FAILURE; } if (!ExistsDir(output_path)) { std::cerr << "ERROR: `output_path` is not a directory" << std::endl; return EXIT_FAILURE; } PrintHeading1("Loading model"); Reconstruction reconstruction; reconstruction.Read(input_path); return RunPointTriangulatorImpl( reconstruction, *options.database_path, *options.image_path, output_path, *options.mapper, clear_points); } int RunPointTriangulatorImpl(Reconstruction& reconstruction, const std::string database_path, const std::string image_path, const std::string output_path, const IncrementalMapperOptions& mapper_options, const bool clear_points) { PrintHeading1("Loading database"); DatabaseCache database_cache; { Timer timer; timer.Start(); Database database(database_path); const size_t min_num_matches = static_cast(mapper_options.min_num_matches); database_cache.Load(database, min_num_matches, mapper_options.ignore_watermarks, mapper_options.image_names); if (clear_points) { reconstruction.DeleteAllPoints2DAndPoints3D(); reconstruction.TranscribeImageIdsToDatabase(database); } std::cout << std::endl; timer.PrintMinutes(); } std::cout << std::endl; CHECK_GE(reconstruction.NumRegImages(), 2) << "Need at least two images for triangulation"; IncrementalMapper mapper(&database_cache); mapper.BeginReconstruction(&reconstruction); ////////////////////////////////////////////////////////////////////////////// // Triangulation ////////////////////////////////////////////////////////////////////////////// const auto tri_options = mapper_options.Triangulation(); const auto& reg_image_ids = reconstruction.RegImageIds(); for (size_t i = 0; i < reg_image_ids.size(); ++i) { const image_t image_id = reg_image_ids[i]; const auto& image = reconstruction.Image(image_id); PrintHeading1(StringPrintf("Triangulating image #%d (%d)", image_id, i)); const size_t num_existing_points3D = image.NumPoints3D(); std::cout << " => Image sees " << num_existing_points3D << " / " << image.NumObservations() << " points" << std::endl; mapper.TriangulateImage(tri_options, image_id); std::cout << " => Triangulated " << (image.NumPoints3D() - num_existing_points3D) << " points" << std::endl; } ////////////////////////////////////////////////////////////////////////////// // Retriangulation ////////////////////////////////////////////////////////////////////////////// PrintHeading1("Retriangulation"); CompleteAndMergeTracks(mapper_options, &mapper); ////////////////////////////////////////////////////////////////////////////// // Bundle adjustment ////////////////////////////////////////////////////////////////////////////// auto ba_options = mapper_options.GlobalBundleAdjustment(); ba_options.refine_focal_length = false; ba_options.refine_principal_point = false; ba_options.refine_extra_params = false; ba_options.refine_extrinsics = false; // Configure bundle adjustment. BundleAdjustmentConfig ba_config; for (const image_t image_id : reconstruction.RegImageIds()) { ba_config.AddImage(image_id); } for (int i = 0; i < mapper_options.ba_global_max_refinements; ++i) { // Avoid degeneracies in bundle adjustment. reconstruction.FilterObservationsWithNegativeDepth(); const size_t num_observations = reconstruction.ComputeNumObservations(); PrintHeading1("Bundle adjustment"); BundleAdjuster bundle_adjuster(ba_options, ba_config); CHECK(bundle_adjuster.Solve(&reconstruction)); size_t num_changed_observations = 0; num_changed_observations += CompleteAndMergeTracks(mapper_options, &mapper); num_changed_observations += FilterPoints(mapper_options, &mapper); const double changed = static_cast(num_changed_observations) / num_observations; std::cout << StringPrintf(" => Changed observations: %.6f", changed) << std::endl; if (changed < mapper_options.ba_global_max_refinement_change) { break; } } PrintHeading1("Extracting colors"); reconstruction.ExtractColorsForAllImages(image_path); const bool kDiscardReconstruction = false; mapper.EndReconstruction(kDiscardReconstruction); reconstruction.Write(output_path); return EXIT_SUCCESS; } namespace { // Read the configuration of the camera rigs from a JSON file. The input images // of a camera rig must be named consistently to assign them to the appropriate // camera rig and the respective snapshots. // // An example configuration of a single camera rig: // [ // { // "ref_camera_id": 1, // "cameras": // [ // { // "camera_id": 1, // "image_prefix": "left1_image" // "rel_tvec": [0, 0, 0], // "rel_qvec": [1, 0, 0, 0] // }, // { // "camera_id": 2, // "image_prefix": "left2_image" // "rel_tvec": [0, 0, 0], // "rel_qvec": [0, 1, 0, 0] // }, // { // "camera_id": 3, // "image_prefix": "right1_image" // "rel_tvec": [0, 0, 0], // "rel_qvec": [0, 0, 1, 0] // }, // { // "camera_id": 4, // "image_prefix": "right2_image" // "rel_tvec": [0, 0, 0], // "rel_qvec": [0, 0, 0, 1] // } // ] // } // ] // // The "camera_id" and "image_prefix" fields are required, whereas the // "rel_tvec" and "rel_qvec" fields optionally specify the relative // extrinsics of the camera rig in the form of a translation vector and a // rotation quaternion. The relative extrinsics rel_qvec and rel_tvec transform // coordinates from rig to camera coordinate space. If the relative extrinsics // are not provided then they are automatically inferred from the // reconstruction. // // This file specifies the configuration for a single camera rig and that you // could potentially define multiple camera rigs. The rig is composed of 4 // cameras: all images of the first camera must have "left1_image" as a name // prefix, e.g., "left1_image_frame000.png" or "left1_image/frame000.png". // Images with the same suffix ("_frame000.png" and "/frame000.png") are // assigned to the same snapshot, i.e., they are assumed to be captured at the // same time. Only snapshots with the reference image registered will be added // to the bundle adjustment problem. The remaining images will be added with // independent poses to the bundle adjustment problem. The above configuration // could have the following input image file structure: // // /path/to/images/... // left1_image/... // frame000.png // frame001.png // frame002.png // ... // left2_image/... // frame000.png // frame001.png // frame002.png // ... // right1_image/... // frame000.png // frame001.png // frame002.png // ... // right2_image/... // frame000.png // frame001.png // frame002.png // ... // std::vector ReadCameraRigConfig(const std::string& rig_config_path, const Reconstruction& reconstruction, bool estimate_rig_relative_poses) { boost::property_tree::ptree pt; boost::property_tree::read_json(rig_config_path.c_str(), pt); std::vector camera_rigs; for (const auto& rig_config : pt) { CameraRig camera_rig; std::vector image_prefixes; for (const auto& camera : rig_config.second.get_child("cameras")) { const int camera_id = camera.second.get("camera_id"); image_prefixes.push_back(camera.second.get("image_prefix")); Eigen::Vector3d rel_tvec; Eigen::Vector4d rel_qvec; int index = 0; auto rel_tvec_node = camera.second.get_child_optional("rel_tvec"); if (rel_tvec_node) { for (const auto& node : rel_tvec_node.get()) { rel_tvec[index++] = node.second.get_value(); } } else { estimate_rig_relative_poses = true; } index = 0; auto rel_qvec_node = camera.second.get_child_optional("rel_qvec"); if (rel_qvec_node) { for (const auto& node : rel_qvec_node.get()) { rel_qvec[index++] = node.second.get_value(); } } else { estimate_rig_relative_poses = true; } camera_rig.AddCamera(camera_id, rel_qvec, rel_tvec); } camera_rig.SetRefCameraId(rig_config.second.get("ref_camera_id")); std::unordered_map> snapshots; for (const auto image_id : reconstruction.RegImageIds()) { const auto& image = reconstruction.Image(image_id); for (const auto& image_prefix : image_prefixes) { if (StringContains(image.Name(), image_prefix)) { const std::string image_suffix = StringGetAfter(image.Name(), image_prefix); snapshots[image_suffix].push_back(image_id); } } } for (const auto& snapshot : snapshots) { bool has_ref_camera = false; for (const auto image_id : snapshot.second) { const auto& image = reconstruction.Image(image_id); if (image.CameraId() == camera_rig.RefCameraId()) { has_ref_camera = true; } } if (has_ref_camera) { camera_rig.AddSnapshot(snapshot.second); } } camera_rig.Check(reconstruction); if (estimate_rig_relative_poses) { PrintHeading2("Estimating relative rig poses"); if (!camera_rig.ComputeRelativePoses(reconstruction)) { std::cout << "WARN: Failed to estimate rig poses from reconstruction; " "cannot use rig BA" << std::endl; return std::vector(); } } camera_rigs.push_back(camera_rig); } return camera_rigs; } } // namespace int RunRigBundleAdjuster(int argc, char** argv) { std::string input_path; std::string output_path; std::string rig_config_path; bool estimate_rig_relative_poses = true; RigBundleAdjuster::Options rig_ba_options; OptionManager options; options.AddRequiredOption("input_path", &input_path); options.AddRequiredOption("output_path", &output_path); options.AddRequiredOption("rig_config_path", &rig_config_path); options.AddDefaultOption("estimate_rig_relative_poses", &estimate_rig_relative_poses); options.AddDefaultOption("RigBundleAdjustment.refine_relative_poses", &rig_ba_options.refine_relative_poses); options.AddBundleAdjustmentOptions(); options.Parse(argc, argv); Reconstruction reconstruction; reconstruction.Read(input_path); PrintHeading1("Camera rig configuration"); auto camera_rigs = ReadCameraRigConfig(rig_config_path, reconstruction, estimate_rig_relative_poses); BundleAdjustmentConfig config; for (size_t i = 0; i < camera_rigs.size(); ++i) { const auto& camera_rig = camera_rigs[i]; PrintHeading2(StringPrintf("Camera Rig %d", i + 1)); std::cout << StringPrintf("Cameras: %d", camera_rig.NumCameras()) << std::endl; std::cout << StringPrintf("Snapshots: %d", camera_rig.NumSnapshots()) << std::endl; // Add all registered images to the bundle adjustment configuration. for (const auto image_id : reconstruction.RegImageIds()) { config.AddImage(image_id); } } PrintHeading1("Rig bundle adjustment"); BundleAdjustmentOptions ba_options = *options.bundle_adjustment; ba_options.solver_options.minimizer_progress_to_stdout = true; RigBundleAdjuster bundle_adjuster(ba_options, rig_ba_options, config); CHECK(bundle_adjuster.Solve(&reconstruction, &camera_rigs)); reconstruction.Write(output_path); return EXIT_SUCCESS; } } // namespace colmap