### Export CMake Targets and Install Configuration Source: https://github.com/jczarnowski/deepfactors/blob/master/CMakeLists.txt This CMake script exports the project's targets for use by other projects and installs configuration files. It defines a namespace for the exported targets and specifies the destination for the configuration files within the installation directory. The `export(PACKAGE)` command makes the package available for `find_package`. ```cmake export(EXPORT ${PROJECT_NAME}Targets FILE "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Targets.cmake" NAMESPACE ${PROJECT_NAME}::) configure_file(${PROJECT_NAME}Config.cmake.in "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake" COPYONLY) set(ConfigPackageLocation lib/cmake/${PROJECT_NAME}) install(EXPORT ${PROJECT_NAME}Targets FILE ${PROJECT_NAME}Targets.cmake NAMESPACE ${PROJECT_NAME}:: DESTINATION ${ConfigPackageLocation}) install( FILES "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake" "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake" DESTINATION ${ConfigPackageLocation} ) # export package to cmake cache export(PACKAGE ${PROJECT_NAME}) ``` -------------------------------- ### Command Line Demo Application Execution Source: https://context7.com/jczarnowski/deepfactors/llms.txt Provides shell commands for building the project and running the demo application with various inputs, including live OpenNI cameras, TUM RGB-D datasets, and ScanNet sequences. ```bash mkdir build && cd build cmake .. make -j8 bash scripts/download_network.bash bash scripts/run_scannet.bash --scene_id scene0356_02 ./bin/df_demo --flagfile=data/flags/live_odom.flags --source_url=openni://0 ./bin/df_demo --flagfile=data/flags/dataset_odom.flags --source_url=tum:///path/to/sequence ./bin/df_demo --flagfile=data/flags/dataset_odom.flags --source_url=scannet:///path/to/scene ./bin/df_demo --flagfile=data/flags/live_refine.flags --source_url=openni://0 ``` -------------------------------- ### Initialize and Run Mapper for Factor Graph Optimization Source: https://context7.com/jczarnowski/deepfactors/llms.txt Shows how to instantiate the Mapper class, configure ISAM2 parameters, and perform incremental optimization. It covers initializing the mapper, adding keyframes, and executing the mapping loop. ```cpp #include "mapper.h" #include "decoder_network.h" auto netcfg = df::LoadJsonNetworkConfig("data/nets/scannet256_32.cfg"); auto network = std::make_shared(netcfg); df::CameraPyramid cam_pyr(camera, netcfg.pyramid_levels); df::MapperOptions mapper_opts; mapper_opts.code_prior = 1.0; mapper_opts.pose_prior = 0.3; mapper_opts.predict_code = true; mapper_opts.use_photometric = true; mapper_opts.use_reprojection = true; mapper_opts.connection_mode = df::MapperOptions::LAST; mapper_opts.max_back_connections = 4; mapper_opts.isam_params.factorization = gtsam::ISAM2Params::CHOLESKY; mapper_opts.isam_params.relinearizeSkip = 1; mapper_opts.isam_params.relinearizeThreshold = 0.05; df::Mapper mapper(mapper_opts, cam_pyr, network); mapper.SetMapCallback([](df::Map::Ptr map) { for (auto& kf : map->keyframes) { std::cout << "Keyframe " << kf.first << " pose: " << kf.second->pose_wk.translation().transpose() << std::endl; } }); // Initialize and enqueue keyframes mapper.InitOneFrame(timestamp, img_gray, img_color, features); auto kf = mapper.EnqueueKeyframe(timestamp, img_gray, img_color, pose_init, features); // Perform optimization while (mapper.HasWork()) { mapper.MappingStep(); } ``` -------------------------------- ### Configure DeepFactors SLAM System Options Source: https://context7.com/jczarnowski/deepfactors/llms.txt Demonstrates how to initialize and configure the DeepFactorsOptions struct. This includes setting paths, tracking parameters, keyframe management, and error function weights for the SLAM system. ```cpp #include "deepfactors_options.h" df::DeepFactorsOptions opts; // Required paths opts.gpu = 0; opts.network_path = "data/nets/scannet256_32.cfg"; opts.vocabulary_path = "data/voc/small_voc.yml.gz"; // Camera tracking configuration opts.tracking_iterations = {4, 6, 5, 10}; opts.tracking_mode = df::DeepFactorsOptions::CLOSEST; opts.tracking_huber_delta = 0.3f; opts.tracking_error_threshold = 0.3f; opts.tracking_dist_threshold = 2.0f; // Keyframe management opts.keyframe_mode = df::DeepFactorsOptions::AUTO; opts.inlier_threshold = 0.5f; opts.dist_threshold = 2.0f; opts.frame_dist_threshold = 0.2f; opts.connection_mode = df::MapperOptions::LAST; opts.max_back_connections = 4; // Loop closure settings opts.loop_closure = true; opts.loop_max_dist = 0.5f; opts.loop_active_window = 10; opts.loop_sigma = 1.0f; opts.loop_min_similarity = 0.35f; opts.loop_max_candidates = 10; // Mapping parameters opts.interleave_mapping = false; opts.relinearize_skip = 1; opts.relinearize_threshold = 0.05f; opts.pose_prior = 0.3f; opts.code_prior = 1.0f; opts.predict_code = true; // Photometric error (dense) opts.use_photometric = true; opts.pho_iters = {15, 15, 15, 30}; opts.huber_delta = 0.3f; opts.sfm_step_threads = 32; opts.sfm_step_blocks = 11; opts.sfm_eval_threads = 224; opts.sfm_eval_blocks = 66; // Reprojection error (sparse features) opts.use_reprojection = false; opts.rep_nfeatures = 500; opts.rep_max_dist = 30.0f; opts.rep_huber = 0.1f; opts.rep_sigma = 1.0f; // Geometric error (sparse depth) opts.use_geometric = false; opts.geo_npoints = 500; opts.geo_huber = 0.1f; ``` -------------------------------- ### Initialize and Run DeepFactors SLAM Source: https://context7.com/jczarnowski/deepfactors/llms.txt This snippet demonstrates the initialization of the DeepFactors SLAM system, including camera configuration, setting options for GPU and network paths, and the main processing loop for tracking and mapping. ```cpp #include "deepfactors.h" #include "pinhole_camera.h" #include constexpr int CODE_SIZE = 32; df::DeepFactors slam; df::PinholeCamera camera(525.0f, 525.0f, 319.5f, 239.5f, 640.0f, 480.0f); df::DeepFactorsOptions opts; opts.gpu = 0; opts.network_path = "data/nets/scannet256_32.cfg"; opts.vocabulary_path = "data/voc/small_voc.yml.gz"; opts.tracking_iterations = {5, 5, 10}; opts.keyframe_mode = df::DeepFactorsOptions::AUTO; opts.use_photometric = true; opts.use_reprojection = true; opts.loop_closure = true; slam.Init(camera, opts); slam.SetPoseCallback([](const Sophus::SE3f& pose_wc) { std::cout << "Camera pose: " << pose_wc.translation().transpose() << std::endl; }); slam.SetMapCallback([](df::Map::Ptr map) { std::cout << "Map updated with " << map->NumKeyframes() << " keyframes" << std::endl; }); cv::Mat first_frame = cv::imread("frame0.jpg"); slam.BootstrapOneFrame(0.0, first_frame); for (int i = 1; i < num_frames; ++i) { cv::Mat frame = cv::imread("frame" + std::to_string(i) + ".jpg"); double timestamp = i * 0.033; slam.ProcessFrame(timestamp, frame); Sophus::SE3f pose = slam.GetCameraPose(); } slam.SaveResults("output/"); ``` -------------------------------- ### Initialize and Execute DecoderNetwork Source: https://context7.com/jczarnowski/deepfactors/llms.txt Demonstrates how to load network configurations, prepare input buffers, and perform depth decoding using the DecoderNetwork class. It covers both standard decoding and combined prediction-decoding workflows. ```cpp #include "decoder_network.h" #include #include df::DecoderNetwork::NetworkConfig cfg = df::LoadJsonNetworkConfig("data/nets/scannet256_32.cfg"); df::DecoderNetwork network(cfg); cv::Mat input_cv = cv::imread("image.jpg", cv::IMREAD_GRAYSCALE); cv::Mat input_float; input_cv.convertTo(input_float, CV_32FC1, 1.0/255.0); vc::Image2DManaged image(cfg.input_width, cfg.input_height); vc::Image2DView image_view(input_float); image.copyFrom(image_view); Eigen::MatrixXf code = Eigen::MatrixXf::Zero(cfg.code_size, 1); vc::RuntimeBufferPyramidManaged prx_out(cfg.pyramid_levels, cfg.input_width, cfg.input_height); vc::RuntimeBufferPyramidManaged stdev_out(cfg.pyramid_levels, cfg.input_width, cfg.input_height); vc::RuntimeBufferPyramidManaged jac_out(cfg.pyramid_levels, cfg.code_size * cfg.input_width, cfg.input_height); network.Decode(image, code, &prx_out, &stdev_out, &jac_out); ``` -------------------------------- ### Perform Camera Tracking with CameraTracker Source: https://context7.com/jczarnowski/deepfactors/llms.txt Shows how to configure the CameraTracker, set a reference keyframe, and perform dense image alignment. It also demonstrates how to retrieve pose estimates and evaluate tracking quality. ```cpp #include "camera_tracker.h" #include "keyframe.h" df::CameraTracker::TrackerConfig tracker_cfg; tracker_cfg.pyramid_levels = 4; tracker_cfg.iterations_per_level = {10, 5, 4, 3}; df::CameraPyramid cam_pyr(camera, tracker_cfg.pyramid_levels); df::CameraTracker tracker(cam_pyr, tracker_cfg); auto keyframe = std::make_shared>(tracker_cfg.pyramid_levels, width, height, code_size); tracker.SetKeyframe(keyframe); vc::RuntimeBufferPyramidManaged pyr_live_img(tracker_cfg.pyramid_levels, width, height); vc::RuntimeBufferPyramidManaged, vc::TargetDeviceCUDA> pyr_live_grad(tracker_cfg.pyramid_levels, width, height); tracker.TrackFrame(pyr_live_img, pyr_live_grad); Sophus::SE3f pose_wc = tracker.GetPoseEstimate(); float inliers = tracker.GetInliers(); float error = tracker.GetError(); ``` -------------------------------- ### Configure CMake Project and Build Options Source: https://github.com/jczarnowski/deepfactors/blob/master/CMakeLists.txt Initializes the project with CXX and CUDA support, sets the build type, and defines custom build options for tests, demos, and system features. ```cmake cmake_minimum_required(VERSION 3.10 FATAL_ERROR) project(deepfactors LANGUAGES CXX CUDA) set(CMAKE_CXX_STANDARD 17) option(DF_BUILD_TESTS "Build unit tests" ON) set(DF_CODE_SIZE 32 CACHE STRING "Size of the compact code representation") add_definitions(-DDF_CODE_SIZE=${DF_CODE_SIZE}) ``` -------------------------------- ### PinholeCamera Model Projection and Jacobian Operations Source: https://context7.com/jczarnowski/deepfactors/llms.txt Demonstrates how to initialize a PinholeCamera, perform 3D-to-2D projection and 2D-to-3D reprojection, and compute associated Jacobians. It also covers utility functions like viewport resizing and intrinsic matrix access. ```cpp #include "pinhole_camera.h" df::PinholeCamera camera(525.0f, 525.0f, 319.5f, 239.5f, 640.0f, 480.0f); df::PinholeCamera camera = df::PinholeCamera::FromFile("camera_calib.yml"); Eigen::Vector3f point_3d(1.0f, 2.0f, 5.0f); Eigen::Vector2f pixel = camera.Project(point_3d); Eigen::Matrix proj_jac = camera.ProjectPointJacobian(point_3d); Eigen::Vector2f pixel(320.0f, 240.0f); float depth = 2.5f; Eigen::Vector3f point = camera.Reproject(pixel, depth); Eigen::Matrix reproj_pixel_jac = camera.ReprojectPixelJacobian(pixel, depth); Eigen::Matrix reproj_depth_jac = camera.ReprojectDepthJacobian(pixel, depth); bool valid = camera.PixelValid(pixel); bool valid_with_border = camera.PixelValid(pixel, 10); camera.ResizeViewport(1280, 960); Eigen::Matrix3f K = camera.Matrix(); Eigen::Matrix3f K_inv = camera.InverseMatrix(); df::PinholeCamera camera_double = camera.Cast(); ``` -------------------------------- ### DeepFactors Demo Configuration Source: https://context7.com/jczarnowski/deepfactors/llms.txt Command-line arguments for running the df_demo script, specifying source data, network paths, GPU usage, and various optimization parameters. This configuration is used to set up and run the SLAM system. ```bash ./bin/df_demo --source_url=scannet:///data/scene0356_02 --network_path=data/nets/scannet256_32.cfg --vocab_path=data/voc/small_voc.yml.gz --calib_path=data/calib/camera.yml --gpu=0 --keyframe_mode=AUTO --tracking_mode=CLOSEST --connection_mode=LAST --use_photometric=true --use_reprojection=true --loop_closure=true --tracking_iters=5,5,10 --pho_iters=15,15,15,30 --huber_delta=0.1 --inlier_threshold=0.5 --dist_threshold=2.0 --run_log_dir=results --init_on_start=true ``` -------------------------------- ### Create Core Brisk Wrapper Library (CMake) Source: https://github.com/jczarnowski/deepfactors/blob/master/sources/core/CMakeLists.txt Defines a shared library 'core_brisk_wrap' to encapsulate Brisk-specific code. This library is compiled with native architecture optimizations (-march=native, -mtune=native) to address compatibility issues with Eigen in GT-SAM. It links against the 'brisk', 'DBoW2', and 'opencv_core' libraries and includes system and Brisk-specific header directories. ```cmake add_library(core_brisk_wrap SHARED system/fbrisk.cpp system/fbrisk.h) target_compile_options(core_brisk_wrap PRIVATE -march=native -mtune=native) target_link_libraries(core_brisk_wrap PUBLIC brisk DBoW2 opencv_core) target_include_directories(core_brisk_wrap PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/system ${BRISK_INCLUDE_DIRS} ) ``` -------------------------------- ### CMake: Configure df_drivers Library Source: https://github.com/jczarnowski/deepfactors/blob/master/sources/drivers/CMakeLists.txt This CMake script configures the df_drivers shared library. It defines source and header files, conditionally includes Point Grey camera drivers if DF_WITH_FLYCAP is enabled, sets public include directories, and links against necessary external libraries. ```cmake find_package(CameraDrivers REQUIRED) set(drivers_sources # camera camera/openni_interface.cpp # dataset dataset/file_interface.cpp dataset/icl_interface.cpp dataset/scannet_interface.cpp dataset/tum_interface.cpp # top-level camera_interface_factory.cpp ) set(drivers_headers # camera camera/live_interface.h camera/openni_interface.h # dataset dataset/dataset_interface.h dataset/file_interface.h dataset/icl_interface.h dataset/scannet_interface.h dataset/tum_interface.h # top-level camera_interface.h camera_interface_factory.h ) if (DF_WITH_FLYCAP) list(APPEND drivers_sources camera/pointgrey_interface.cpp) list(APPEND drivers_headers camera/pointgrey_interface.h) endif() add_library(df_drivers SHARED ${drivers_sources} ${drivers_headers}) target_include_directories(df_drivers PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/camera ${CMAKE_CURRENT_SOURCE_DIR}/dataset ${CMAKE_CURRENT_SOURCE_DIR} ) target_link_libraries(df_drivers PUBLIC df_common CameraDrivers boost_filesystem glog ) ``` -------------------------------- ### Build Unit Test Executable Source: https://github.com/jczarnowski/deepfactors/blob/master/tests/CMakeLists.txt Creates an executable target named 'ut_df' using the specified source and header files. It also sets include directories and links required libraries. ```cmake add_executable(ut_df ${test_sources} ${test_headers}) target_include_directories(ut_df PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(ut_df PUBLIC gtest glog df_cuda df_core Eigen3::Eigen VisionCore ) ``` -------------------------------- ### Define Test Header Files Source: https://github.com/jczarnowski/deepfactors/blob/master/tests/CMakeLists.txt Lists the header files used by the test executable. These typically include utility headers for testing. ```cmake set(test_headers random_machine.h testing_utils.h) ``` -------------------------------- ### Define Test Source Files Source: https://github.com/jczarnowski/deepfactors/blob/master/tests/CMakeLists.txt Lists the C++ source files that will be compiled as part of the test executable. These files contain the unit tests for various components. ```cmake set(test_sources main.cpp ut_decoder.cpp ut_cuda_utils.cpp ut_se3aligner.cpp ut_sfmaligner.cpp ut_warping.cpp ut_pinhole_camera.cpp ut_nearestpsd.cpp ut_sampling.cpp ) ``` -------------------------------- ### Add Googletest Subdirectory Source: https://github.com/jczarnowski/deepfactors/blob/master/tests/CMakeLists.txt Includes the Googletest framework as a subdirectory within the project, making its targets available for linking. ```cmake add_subdirectory(googletest) ``` -------------------------------- ### Define CMake Library Target and Dependencies Source: https://github.com/jczarnowski/deepfactors/blob/master/sources/common/CMakeLists.txt This snippet defines the source and header files for the df_common library and configures the build target. It sets up include directories and links the necessary external dependencies like Eigen, Sophus, and OpenCV. ```cmake set(common_sources image_sequences.cpp timing.cpp logutils.cpp display_utils.cpp interp.cpp ) set(common_headers algorithm/pinhole_camera_impl.h algorithm/m_estimators.h algorithm/dense_sfm.h algorithm/camera_pyramid.h algorithm/warping.h algorithm/pinhole_camera.h algorithm/nearest_psd.h algorithm/lucas_kanade_se3.h image_sequences.h indexed_map.h tum_io.h interp.h timing.h display_utils.h logutils.h intrinsics.h ) add_library(df_common SHARED ${common_sources} ${common_headers}) target_include_directories(df_common PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/algorithm ) target_link_libraries(df_common PUBLIC Eigen3::Eigen Sophus::Sophus VisionCore opencv_core opencv_highgui jsoncpp ) ``` -------------------------------- ### Manage Project Dependencies Source: https://github.com/jczarnowski/deepfactors/blob/master/CMakeLists.txt Locates required external libraries and packages using CMake's find_package command, ensuring all necessary dependencies are available for the build. ```cmake find_package(CUDA QUIET REQUIRED) find_package(Jsoncpp QUIET REQUIRED) find_package(Sophus QUIET REQUIRED) find_package(OpenCV QUIET REQUIRED) find_package(TensorFlow QUIET REQUIRED) find_package(Eigen3 QUIET REQUIRED) ``` -------------------------------- ### Define GUI Library Build Targets in CMake Source: https://github.com/jczarnowski/deepfactors/blob/master/sources/gui/CMakeLists.txt This snippet configures the df_gui shared library by specifying source files, header files, include directories, and necessary link libraries. It also sets compile definitions for shader directory paths. ```cmake set(gui_sources visualizer.cpp keyframe_renderer.cpp ) set(gui_headers visualizer.h keyframe_renderer.h ) add_library(df_gui SHARED ${gui_sources} ${gui_headers}) target_include_directories(df_gui PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${Pangolin_INCLUDE_DIR} ) target_link_libraries(df_gui PUBLIC df_common df_core gflags glog ${Pangolin_LIBRARY} ) target_compile_definitions(df_gui PUBLIC -DDF_SHADER_DIR="${CMAKE_CURRENT_SOURCE_DIR}/shaders") ``` -------------------------------- ### Define CUDA Source and Header Files Source: https://github.com/jczarnowski/deepfactors/blob/master/sources/cuda/CMakeLists.txt Lists the source (.cpp) and header (.h) files that belong to the CUDA component of the DeepFactors project. These files are used to compile the CUDA library. ```cmake set(cuda_sources cu_sfmaligner.cpp cu_image_proc.cpp cuda_context.cpp cu_se3aligner.cpp cu_depthaligner.cpp ) set(cuda_headers kernel_utils.h cuda_context.h synced_pyramid.h cu_sfmaligner.h launch_utils.h cu_se3aligner.h cu_depthaligner.h device_info.h cu_image_proc.h reduction_items.h ) set_source_files_properties(${cuda_sources} PROPERTIES LANGUAGE CUDA) ``` -------------------------------- ### LoopDetector Place Recognition and Geometric Verification Source: https://context7.com/jczarnowski/deepfactors/llms.txt Shows the configuration and usage of the LoopDetector class for global and local loop closure detection. It includes adding keyframes to the database and enqueuing constraints to the mapper upon successful detection. ```cpp #include "loop_detector.h" df::LoopDetectorConfig loop_cfg; loop_cfg.iters = {5, 5, 10}; loop_cfg.min_similarity = 0.35f; loop_cfg.max_error = 0.5f; loop_cfg.max_dist = 0.1f; loop_cfg.max_candidates = 3; loop_cfg.active_window = 5; df::LoopDetector loop_detector("data/voc/small_voc.yml.gz", map, loop_cfg, cam_pyr); loop_detector.AddKeyframe(keyframe_ptr); df::LoopDetector::LoopInfo loop_info = loop_detector.DetectLoop(pyr_live_img, pyr_live_grad, features, current_keyframe, current_pose); if (loop_info.detected) { std::cout << "Loop detected with keyframe " << loop_info.loop_id << std::endl; mapper.EnqueueLink(current_kf_id, loop_info.loop_id, 1.0f, true, true, false); } int local_loop_id = loop_detector.DetectLocalLoop(pyr_live_img, pyr_live_grad, features, current_keyframe, current_pose); loop_detector.Reset(); ``` -------------------------------- ### Configure External Dependencies with find_package Source: https://github.com/jczarnowski/deepfactors/blob/master/thirdparty/CMakeLists.txt Locates required external libraries using CMake's find_package command. It sets specific search paths for local builds and ensures essential dependencies are available for the project. ```cmake set(INSTALL_DIR "${CMAKE_CURRENT_SOURCE_DIR}/install") set(BUILD_DIR "${CMAKE_CURRENT_SOURCE_DIR}/build") set(CMAKE_PREFIX_PATH ${INSTALL_DIR}) find_package(Sophus NO_DEFAULT_PATH PATHS ${BUILD_DIR}/Sophus REQUIRED) find_package(VisionCore NO_DEFAULT_PATH PATHS ${BUILD_DIR}/vision_core REQUIRED) find_package(brisk PATHS ${INSTALL_DIR}/lib/CMake REQUIRED) find_package(CameraDrivers QUIET REQUIRED) find_package(DBoW2 QUIET REQUIRED) find_package(GTSAM QUIET REQUIRED) find_package(opengv QUIET REQUIRED) find_package(Eigen3 QUIET REQUIRED) find_package(Pangolin QUIET REQUIRED) ``` -------------------------------- ### Add Subdirectories in CMakeLists.txt Source: https://github.com/jczarnowski/deepfactors/blob/master/sources/CMakeLists.txt This snippet demonstrates how to include subdirectories within a CMake project. It uses the `add_subdirectory` command to incorporate modules like 'common', 'core', and 'cuda'. Conditional logic based on CMake variables (e.g., DF_BUILD_DEMO, DF_BUILD_TOOLS) is used to selectively add other modules such as 'demo', 'tools', 'drivers', and 'gui'. ```cmake add_subdirectory(common) add_subdirectory(core) add_subdirectory(cuda) if(DF_BUILD_DEMO) add_subdirectory(demo) endif() if(DF_BUILD_TOOLS) add_subdirectory(tools) endif() if(DF_BUILD_TOOLS OR DF_BUILD_DEMO) add_subdirectory(drivers) add_subdirectory(gui) endif() ``` -------------------------------- ### PhotometricFactor for Dense Optimization in C++ Source: https://context7.com/jczarnowski/deepfactors/llms.txt Implementation of the PhotometricFactor class using GTSAM for dense photometric error calculation. This factor optimizes poses and depth codes by minimizing the photometric error between frames. ```cpp #include "photometric_factor.h" #include // Factor connects two poses and one code gtsam::Key pose0_key = gtsam::Symbol('p', 0); gtsam::Key pose1_key = gtsam::Symbol('p', 1); gtsam::Key code0_key = gtsam::Symbol('c', 0); // Create photometric factor at pyramid level int pyrlevel = 0; // Finest level auto factor = boost::make_shared>( camera, // Pinhole camera model keyframe0, // Reference keyframe (has depth) frame1, // Target frame (warped to reference) pose0_key, pose1_key, code0_key, pyrlevel, sfm_aligner, // CUDA warping/alignment module true // Update validity mask ); // Create GTSAM values gtsam::Values values; values.insert(pose0_key, keyframe0->pose_wk); values.insert(pose1_key, frame1->pose_wk); values.insert(code0_key, keyframe0->code); // Compute error double error = factor->error(values); // Linearize to get Gaussian factor (Hessian form) auto gaussian_factor = factor->linearize(values); // Factor dimension std::cout << "Factor dim: " << factor->dim() << std::endl; // 2*6 + 32 = 44 // Factor info std::cout << factor->Name() << std::endl; // e.g., "PhotoFactor(kf0->fr1@lvl0)" ``` -------------------------------- ### DeepFactors SLAM Interface Source: https://context7.com/jczarnowski/deepfactors/llms.txt The DeepFactors class is the main entry point for the SLAM system, managing camera intrinsics, configuration options, and real-time frame processing. ```APIDOC ## DeepFactors Class Interface ### Description The DeepFactors class manages the lifecycle of the SLAM system, including initialization, tracking, and map updates. ### Method C++ Class Instance ### Parameters #### Configuration Options (DeepFactorsOptions) - **gpu** (int) - Required - GPU device ID to use for neural network inference. - **network_path** (string) - Required - Path to the TensorFlow-based neural network configuration file. - **vocabulary_path** (string) - Required - Path to the ORB vocabulary file for loop closure. - **tracking_iterations** (vector) - Optional - Number of iterations for the image pyramid levels. ### Usage Example ```cpp df::DeepFactors slam; df::DeepFactorsOptions opts; opts.gpu = 0; slam.Init(camera, opts); slam.BootstrapOneFrame(timestamp, frame); slam.ProcessFrame(timestamp, frame); ``` ### Callbacks - **SetPoseCallback**: Triggered when the camera pose is updated. - **SetMapCallback**: Triggered when the keyframe map is updated. ``` -------------------------------- ### DecoderNetwork - Neural Network Depth Decoder Source: https://context7.com/jczarnowski/deepfactors/llms.txt This section details the usage of the DecoderNetwork class to decode compact code representations into dense depth maps, Jacobians, and uncertainty estimates using a TensorFlow model. ```APIDOC ## DecoderNetwork - Neural Network Depth Decoder ### Description The `DecoderNetwork` class interfaces with a TensorFlow model to decode compact code representations into dense depth maps with associated Jacobians and uncertainty estimates. ### Method `Decode(image, code, &prx_out, &stdev_out, &jac_out)` ### Endpoint N/A (Class method) ### Parameters #### Request Body - **image** (vc::Image2DManaged) - Input image buffer. - **code** (Eigen::MatrixXf) - Compact code representation. - **prx_out** (vc::RuntimeBufferPyramidManaged*) - Output buffer for proximity maps. - **stdev_out** (vc::RuntimeBufferPyramidManaged*) - Output buffer for standard deviation maps. - **jac_out** (vc::RuntimeBufferPyramidManaged*) - Output buffer for Jacobian matrices. ### Request Example ```cpp // Load network configuration from JSON df::DecoderNetwork::NetworkConfig cfg = df::LoadJsonNetworkConfig("data/nets/scannet256_32.cfg"); // Create decoder network df::DecoderNetwork network(cfg); // Prepare input image cv::Mat input_cv = cv::imread("image.jpg", cv::IMREAD_GRAYSCALE); cv::Mat input_float; input_cv.convertTo(input_float, CV_32FC1, 1.0/255.0); vc::Image2DManaged image(cfg.input_width, cfg.input_height); vc::Image2DView image_view(input_float); image.copyFrom(image_view); // Initialize code Eigen::MatrixXf code = Eigen::MatrixXf::Zero(cfg.code_size, 1); // Allocate output pyramids vc::RuntimeBufferPyramidManaged prx_out(cfg.pyramid_levels, cfg.input_width, cfg.input_height); vc::RuntimeBufferPyramidManaged stdev_out(cfg.pyramid_levels, cfg.input_width, cfg.input_height); vc::RuntimeBufferPyramidManaged jac_out(cfg.pyramid_levels, cfg.code_size * cfg.input_width, cfg.input_height); // Decode network.Decode(image, code, &prx_out, &stdev_out, &jac_out); ``` ### Response #### Success Response (200) - **prx_out** (vc::RuntimeBufferPyramidManaged) - Populated proximity maps. - **stdev_out** (vc::RuntimeBufferPyramidManaged) - Populated uncertainty maps. - **jac_out** (vc::RuntimeBufferPyramidManaged) - Populated Jacobian matrices. #### Response Example N/A (Outputs are passed by pointer) --- ### Method `PredictAndDecode(image, code, &predicted_code, &prx_out, &stdev_out, &jac_out)` ### Endpoint N/A (Class method) ### Parameters #### Request Body - **image** (vc::Image2DManaged) - Input image buffer. - **code** (Eigen::MatrixXf) - Input code representation. - **predicted_code** (Eigen::MatrixXf*) - Output for the predicted code. - **prx_out** (vc::RuntimeBufferPyramidManaged*) - Output buffer for proximity maps. - **stdev_out** (vc::RuntimeBufferPyramidManaged*) - Output buffer for uncertainty maps. - **jac_out** (vc::RuntimeBufferPyramidManaged*) - Output buffer for Jacobian matrices. ### Request Example ```cpp // ... (previous setup code for network, image, code, and output pyramids) ... Eigen::MatrixXf predicted_code; network.PredictAndDecode(image, code, &predicted_code, &prx_out, &stdev_out, &jac_out); ``` ### Response #### Success Response (200) - **predicted_code** (Eigen::MatrixXf) - The predicted code. - **prx_out** (vc::RuntimeBufferPyramidManaged) - Populated proximity maps. - **stdev_out** (vc::RuntimeBufferPyramidManaged) - Populated uncertainty maps. - **jac_out** (vc::RuntimeBufferPyramidManaged) - Populated Jacobian matrices. #### Response Example N/A (Outputs are passed by pointer) --- ### Description Converts proximity values to depth using the average depth normalization factor. ### Method Accessing Proximity Map Level ### Endpoint N/A (Direct access to pyramid level) ### Parameters None ### Request Example ```cpp // Convert proximity to depth: depth = avg_dpt / proximity // Access level 0 (finest resolution) auto prx_level0 = prx_out[0]; // depth = cfg.avg_dpt / prx_level0 ``` ### Response #### Success Response (200) - **prx_level0** (auto) - Access to the finest resolution proximity map. #### Response Example N/A ``` -------------------------------- ### Define Imported Library Targets Source: https://github.com/jczarnowski/deepfactors/blob/master/thirdparty/CMakeLists.txt Manually creates an imported library target for DBoW2 to resolve issues with non-standard configuration files. This ensures the library's location and include directories are correctly exposed to the build system. ```cmake add_library(DBoW2 SHARED IMPORTED GLOBAL) set_property(TARGET DBoW2 APPEND PROPERTY IMPORTED_LOCATION ${DBoW2_LIBRARY}) set_property(TARGET DBoW2 APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${DBoW2_INCLUDE_DIR}) ``` -------------------------------- ### CameraTracker - Frame-to-Keyframe Tracking API Source: https://context7.com/jczarnowski/deepfactors/llms.txt This section describes the CameraTracker class, which performs dense direct image alignment to estimate camera pose relative to a reference keyframe using a pyramid scheme. ```APIDOC ## CameraTracker - Frame-to-Keyframe Tracking ### Description The `CameraTracker` class performs dense direct image alignment to estimate the 6-DOF camera pose relative to a reference keyframe using a coarse-to-fine pyramid scheme. ### Method `TrackFrame(pyr_live_img, pyr_live_grad)` ### Endpoint N/A (Class method) ### Parameters #### Request Body - **pyr_live_img** (vc::RuntimeBufferPyramidManaged) - Pyramid of live frame images. - **pyr_live_grad** (vc::RuntimeBufferPyramidManaged, vc::TargetDeviceCUDA>) - Pyramid of live frame image gradients. ### Request Example ```cpp #include "camera_tracker.h" #include "keyframe.h" // Configure tracker df::CameraTracker::TrackerConfig tracker_cfg; tracker_cfg.pyramid_levels = 4; tracker_cfg.iterations_per_level = {10, 5, 4, 3}; tracker_cfg.huber_delta = 0.1; // Create tracker with camera pyramid df::CameraPyramid cam_pyr(camera, tracker_cfg.pyramid_levels); df::CameraTracker tracker(cam_pyr, tracker_cfg); // Set reference keyframe auto keyframe = std::make_shared>( tracker_cfg.pyramid_levels, width, height, code_size); // ... populate keyframe with image, depth, pose ... tracker.SetKeyframe(keyframe); // Prepare live frame pyramids vc::RuntimeBufferPyramidManaged pyr_live_img(tracker_cfg.pyramid_levels, width, height); vc::RuntimeBufferPyramidManaged, vc::TargetDeviceCUDA> pyr_live_grad(tracker_cfg.pyramid_levels, width, height); // Upload live frame to GPU pyramids with Gaussian blur downsampling // ... populate pyramids ... // Track current frame against keyframe tracker.TrackFrame(pyr_live_img, pyr_live_grad); ``` ### Response #### Success Response (200) - **Pose Estimate**: `tracker.GetPoseEstimate()` returns the estimated 6-DOF camera pose (Sophus::SE3f). - **Inliers**: `tracker.GetInliers()` returns the ratio of inlier pixels (0-1). - **Error**: `tracker.GetError()` returns the mean photometric error. - **Residual Image**: `tracker.GetResidualImage()` returns the residual image. #### Response Example ```cpp Sophus::SE3f pose_wc = tracker.GetPoseEstimate(); float inliers = tracker.GetInliers(); // Ratio of inlier pixels (0-1) float error = tracker.GetError(); // Mean photometric error cv::Mat residual = tracker.GetResidualImage(); // Check tracking quality if (inliers < 0.5 || error > 0.3) { std::cout << "Tracking may be lost!" << std::endl; } ``` --- ### Method `SetKeyframe(keyframe)` ### Endpoint N/A (Class method) ### Parameters #### Request Body - **keyframe** (std::shared_ptr>): The reference keyframe to track against. ### Request Example ```cpp // ... (tracker_cfg setup) ... auto keyframe = std::make_shared>( tracker_cfg.pyramid_levels, width, height, code_size); // ... populate keyframe ... tracker.SetKeyframe(keyframe); ``` ### Response None #### Response Example N/A --- ### Method `SetPose(initial_pose_estimate)` ### Endpoint N/A (Class method) ### Parameters #### Request Body - **initial_pose_estimate** (Sophus::SE3f): The initial pose estimate for the tracker. ### Request Example ```cpp Sophus::SE3f initial_pose_estimate; // ... set initial_pose_estimate ... tracker.SetPose(initial_pose_estimate); ``` ### Response None #### Response Example N/A --- ### Method `Reset()` ### Endpoint N/A (Class method) ### Parameters None ### Request Example ```cpp tracker.Reset(); ``` ### Response None #### Response Example N/A ``` -------------------------------- ### Define Core DeepFactors Library (CMake) Source: https://github.com/jczarnowski/deepfactors/blob/master/sources/core/CMakeLists.txt Defines the main shared library 'df_core' for the DeepFactors project. It includes source and header files from various modules like gtsam, system, features, mapping, and network. The library is configured with specific include directories and links against several other libraries including 'df_common', 'df_cuda', 'Eigen3::Eigen', 'VisionCore', 'DBoW2', 'gtsam', 'opengv', 'glog', 'core_brisk_wrap', 'TensorFlow', and 'jsoncpp'. ```cmake set(core_sources # gtsam gtsam/sparse_geometric_factor.cpp gtsam/photometric_factor.cpp gtsam/reprojection_factor.cpp gtsam/depth_prior_factor.cpp gtsam/uniform_sampler.cpp # system system/camera_tracker.cpp system/loop_detector.cpp # features features/matching.cpp # mapping mapping/work.cpp mapping/work_manager.cpp mapping/mapper.cpp mapping/df_work.cpp # network network/decoder_network.cpp # top-level deepfactors.cpp deepfactors_options.cpp ) set(core_headers # gtsam gtsam/photometric_factor.h gtsam/reprojection_factor.h gtsam/gtsam_traits.h gtsam/sparse_geometric_factor.h gtsam/gtsam_utils.h gtsam/uniform_sampler.h gtsam/factor_graph.h gtsam/depth_prior_factor.h # system system/camera_tracker.h system/loop_detector.h # features features/feature_detection.h features/matching.h # mapping mapping/keyframe_map.h mapping/keyframe.h mapping/work_manager.h mapping/df_work.h mapping/mapper.h mapping/frame.h mapping/work.h # network network/tfwrap.h network/decoder_network.h # top-level deepfactors.h deepfactors_options.h ) add_library(df_core SHARED ${core_sources} ${core_headers}) target_include_directories(df_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/features ${CMAKE_CURRENT_SOURCE_DIR}/gtsam ${CMAKE_CURRENT_SOURCE_DIR}/mapping ${CMAKE_CURRENT_SOURCE_DIR}/network ${CMAKE_CURRENT_SOURCE_DIR}/system ${CMAKE_CURRENT_SOURCE_DIR} ) target_link_libraries(df_core PUBLIC df_common df_cuda Eigen3::Eigen VisionCore DBoW2 gtsam opengv glog core_brisk_wrap PRIVATE TensorFlow jsoncpp ) ``` -------------------------------- ### Keyframe Data Structure in C++ Source: https://context7.com/jczarnowski/deepfactors/llms.txt Implementation of the Keyframe class in C++ for storing keyframe data, including image pyramids, depth maps, feature descriptors, and latent code representations. It supports CPU/GPU synchronization and cloning. ```cpp #include "keyframe.h" #include // Create keyframe with specified dimensions std::size_t pyr_levels = 4; std::size_t width = 256; std::size_t height = 192; std::size_t code_size = 32; auto keyframe = std::make_shared>(pyr_levels, width, height, code_size); // Set keyframe data keyframe->id = 1; keyframe->timestamp = 0.033; keyframe->pose_wk = Sophus::SE3f(); // World-to-keyframe pose keyframe->color_img = color_image; // cv::Mat BGR // Code representation (optimized by GTSAM) keyframe->code = Eigen::VectorXf::Zero(code_size); // Image pyramid (synchronized CPU/GPU) keyframe->pyr_img; // Grayscale image pyramid keyframe->pyr_grad; // Image gradient pyramid // Depth data pyramids keyframe->pyr_dpt; // Depth pyramid keyframe->pyr_vld; // Validity mask pyramid keyframe->pyr_stdev; // Depth uncertainty pyramid keyframe->pyr_prx_orig; // Original proximity (before optimization) keyframe->pyr_jac; // Jacobian of depth w.r.t. code // Feature data keyframe->features.keypoints; // std::vector keyframe->features.descriptors; // cv::Mat (BRISK descriptors) // Access depth at finest level auto& dpt_cpu = keyframe->pyr_dpt.GetCpuLevel(0); float depth_at_pixel = dpt_cpu(x, y); // Upload to GPU keyframe->pyr_dpt.UploadToGpu(); auto& dpt_gpu = keyframe->pyr_dpt.GetGpuLevel(0); // Clone keyframe auto kf_copy = keyframe->Clone(); // Check type bool is_kf = keyframe->IsKeyframe(); // true std::string name = keyframe->Name(); // "kf1" ``` -------------------------------- ### Define CMake Executable Targets Source: https://github.com/jczarnowski/deepfactors/blob/master/sources/tools/CMakeLists.txt Configures build targets for DeepFactors utilities. Each snippet links specific project libraries like df_core, df_cuda, and external dependencies such as OpenCV, Eigen, and DBoW2. ```cmake add_executable(decode_image decode_image.cpp) target_link_libraries(decode_image PUBLIC df_core df_common opencv_highgui opencv_imgproc opencv_core Eigen3::Eigen glog gflags) add_executable(kernel_benchmark kernel_benchmark.cpp) target_include_directories(kernel_benchmark PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(kernel_benchmark PUBLIC df_cuda df_core df_common opencv_highgui opencv_core opencv_imgproc gflags) add_executable(result_viewer result_viewer.cpp) target_include_directories(result_viewer PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(result_viewer PUBLIC df_gui df_drivers opencv_core opencv_highgui gflags glog) add_executable(voc_builder voc_builder.cpp) target_include_directories(voc_builder PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${BRISK_INCLUDE_DIRS}) target_link_libraries(voc_builder PUBLIC df_drivers df_core opencv_core opencv_highgui opencv_features2d brisk gflags glog DBoW2) add_executable(voc_test voc_test.cpp) target_include_directories(voc_test PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${BRISK_INCLUDE_DIRS}) target_link_libraries(voc_test PUBLIC df_core opencv_core opencv_highgui opencv_features2d brisk DBoW2) ``` -------------------------------- ### Configure NVCC Flags for DeepFactors CUDA Source: https://github.com/jczarnowski/deepfactors/blob/master/sources/cuda/CMakeLists.txt Sets and configures NVCC (NVIDIA CUDA Compiler) flags for the DeepFactors project. It includes options for automatic architecture detection, relaxed constexpr, extended lambda, fast math, and suppressing specific warnings. Debug flags are conditionally appended based on the build type. ```cmake set(DF_CUDA_ARCH Auto CACHE STRING "A list of CUDA architectures to compile for. Specifying 'Auto' will attempt to autodetect available GPU devices") CUDA_SELECT_NVCC_ARCH_FLAGS(CUDA_NVCC_ARCH_FLAGS ${DF_CUDA_ARCH}) set(CMAKE_CUDA_FLAGS ${CUDA_NVCC_ARCH_FLAGS};--expt-relaxed-constexpr;--expt-extended-lambda;--use_fast_math) message("Compiling for CUDA architectures: ${CUDA_NVCC_ARCH_FLAGS}") # suppress NVCC warning triggered in Sophus: # "__device__ annotation is ignored on a function that is explicitly defaulted on its first declaration" # see: https://github.com/kokkos/kokkos/issues/1473 list(APPEND CMAKE_CUDA_FLAGS -Xcudafe;--diag_suppress=esa_on_defaulted_function_ignored) # append debug flags if(CMAKE_BUILD_TYPE MATCHES Debug) #list(APPEND CMAKE_CUDA_FLAGS --device-debug;--debug;-Xcompiler;-rdynamic;)#--ptxas-options=-v) elseif(CMAKE_BUILD_TYPE MATCHES RelWithDebInfo) list(APPEND CMAKE_CUDA_FLAGS -g;-lineinfo) endif() string(REPLACE ";" " " _TMP_STR "${CMAKE_CUDA_FLAGS}") set(CMAKE_CUDA_FLAGS "${_TMP_STR}") ```