diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 8281fb33cf..8ad422e733 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -26,6 +26,7 @@ add_subdirectory(save-to-disk) add_subdirectory(multicam) add_subdirectory(pointcloud) add_subdirectory(align) +add_subdirectory(align-gl) add_subdirectory(align-advanced) add_subdirectory(sensor-control) add_subdirectory(measure) @@ -37,4 +38,3 @@ add_subdirectory(record-playback) add_subdirectory(motion) add_subdirectory(gl) add_subdirectory(hdr) - diff --git a/examples/align-gl/CMakeLists.txt b/examples/align-gl/CMakeLists.txt new file mode 100644 index 0000000000..565084e289 --- /dev/null +++ b/examples/align-gl/CMakeLists.txt @@ -0,0 +1,18 @@ +# License: Apache 2.0. See LICENSE file in root directory. +# Copyright(c) 2024 Intel Corporation. All Rights Reserved. +# minimum required cmake version: 3.1.0 +cmake_minimum_required(VERSION 3.1.0) + +project(RealsenseExamplesAlignGl ) + +# Save the command line compile commands in the build output +set(CMAKE_EXPORT_COMPILE_COMMANDS 1) + +if(BUILD_GRAPHICAL_EXAMPLES AND NOT APPLE) + add_executable(rs-align-gl rs-align-gl.cpp ../../third-party/imgui/imgui.cpp ../../third-party/imgui/imgui_draw.cpp ../../third-party/imgui/imgui_impl_glfw.cpp) + set_property(TARGET rs-align-gl PROPERTY CXX_STANDARD 11) + target_link_libraries(rs-align-gl ${DEPENDENCIES} realsense2-gl) + include_directories(../../common ../../third-party/imgui ../../examples) + set_target_properties (rs-align-gl PROPERTIES FOLDER Examples) + install(TARGETS rs-align-gl RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +endif() diff --git a/examples/align-gl/readme.md b/examples/align-gl/readme.md new file mode 100644 index 0000000000..cf4daa65ab --- /dev/null +++ b/examples/align-gl/readme.md @@ -0,0 +1,114 @@ +# rs-align-gl Sample + +## Overview + +> Due to driver limitations built-in GPU capabilities are not available at the moment on Mac-OS + +This example demonstrates the concept of spatial stream alignment **accelerated with GPU**. +For example usecase of alignment, please check out align-advanced and measure demos. +The need for spatial alignment (from here "align") arises from the fact +that not all camera streams are captured from a single viewport. + + +Align process lets the user translate images from one viewport to another. +That said, results of align are synthetic streams, and suffer from several artifacts: +1. **Sampling** - mapping stream to a different viewport will modify the resolution of the frame + to match the resolution of target viewport. This will either cause downsampling or + upsampling via interpolation. The interpolation used needs to be of type + Nearest Neighbor to avoid introducing non-existing values. +2. **Occlussion** - Some pixels in the resulting image correspond to 3D coordinates that the original + sensor did not see, because these 3D points were occluded in the original viewport. + Such pixels may hold invalid texture values. + +## Expected Output + +The application should open a window and display video stream from the color camera overlayed on top of depth stream data. +The slider in bottom of the window control the transparancy of the overlayed stream. +Checkboxes below allow toggling between depth to color vs color to depth alignment. + +

screenshot gif

+ +## Code Overview + +In addition to core `realsense2` this example also depends on an auxiliary `realsense2-gl` library. +This is not strictly part of core RealSense functionality, but rather a useful extension. + +So, in addition to standard `#include ` you need to include: +```cpp +#include // Include GPU-Processing API +``` + +This example also uses `IMGUI` library for simple UI rendering: + +> In order to allow texture sharing between processing and rendering application, `rs_processing_gl.hpp` needs to be included **after** including `GLFW`. + +After defining some helper functions and `window` object, we need to initialize GL processing and rendering subsystems: +```cpp +// Once we have a window, initialize GL module +// Pass our window to enable sharing of textures between processed frames and the window +rs2::gl::init_processing(app, use_gpu_processing); +// Initialize rendering module: +rs2::gl::init_rendering(); +``` + +In this example we are interested in `RS2_STREAM_DEPTH` and `RS2_STREAM_COLOR` streams: +```cpp +// Create a pipeline to easily configure and start the camera +rs2::pipeline pipe; +rs2::config cfg; +cfg.enable_stream(RS2_STREAM_DEPTH); +cfg.enable_stream(RS2_STREAM_COLOR); +pipe.start(cfg); + +``` + +SDK class responsible for GL stream alignment is called `rs2::gl::align`. The user should initialize it with desired target stream and applies it to framesets via `process` method. +```cpp +// Define two align objects. One will be used to align +// to depth viewport and the other to color. +// Creating align object is an expensive operation +// that should not be performed in the main loop +rs2::gl::align align_to_depth(RS2_STREAM_DEPTH); +rs2::gl::align align_to_color(RS2_STREAM_COLOR); +// ... +frameset = align_to_depth.process(frameset); +``` + +This example also uses GL colorizer `rs2::gl::colorizer` to colorize the depth image. +```cpp +rs2::gl::colorizer c; +``` + +Next, we render the two stream overlayed on top of each other using OpenGL blending feature: + +```cpp +glEnable(GL_BLEND); +// Use the Alpha channel for blending +glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); + +if (dir == direction::to_depth) +{ + // When aligning to depth, first render depth image + // and then overlay color on top with transparancy + depth_image.render(colorized_depth, { 0, 0, app.width(), app.height() }); + color_image.render(color, { 0, 0, app.width(), app.height() }, alpha); +} +else +{ + // When aligning to color, first render color image + // and then overlay depth image on top + color_image.render(color, { 0, 0, app.width(), app.height() }); + depth_image.render(colorized_depth, { 0, 0, app.width(), app.height() }, 1 - alpha); +} + +glColor4f(1.f, 1.f, 1.f, 1.f); +glDisable(GL_BLEND); +``` + +IMGUI is used to render the slider and two checkboxes: +```cpp +// Render the UI: +ImGui_ImplGlfw_NewFrame(1); +render_slider({ 15.f, app.height() - 60, app.width() - 30, app.height() }, &alpha, &dir); +ImGui::Render(); +``` \ No newline at end of file diff --git a/examples/align-gl/rs-align-gl.cpp b/examples/align-gl/rs-align-gl.cpp new file mode 100644 index 0000000000..63e4305bcf --- /dev/null +++ b/examples/align-gl/rs-align-gl.cpp @@ -0,0 +1,176 @@ +// License: Apache 2.0. See LICENSE file in root directory. +// Copyright(c) 2024 Intel Corporation. All Rights Reserved. + +#include +#include "example-imgui.hpp" +#include // Include GPU-Processing API + + +/* + This example introduces the concept of spatial stream alignment. + For example usecase of alignment, please check out align-advanced and measure demos. + The need for spatial alignment (from here "align") arises from the fact + that not all camera streams are captured from a single viewport. + Align process lets the user translate images from one viewport to another. + That said, results of align are synthetic streams, and suffer from several artifacts: + 1. Sampling - mapping stream to a different viewport will modify the resolution of the frame + to match the resolution of target viewport. This will either cause downsampling or + upsampling via interpolation. The interpolation used needs to be of type + Nearest Neighbor to avoid introducing non-existing values. + 2. Occlussion - Some pixels in the resulting image correspond to 3D coordinates that the original + sensor did not see, because these 3D points were occluded in the original viewport. + Such pixels may hold invalid texture values. +*/ + +// This example assumes camera with depth and color +// streams, and direction lets you define the target stream +enum class direction +{ + to_depth, + to_color +}; + +// Forward definition of UI rendering, implemented below +void render_slider(rect location, float* alpha, direction* dir); + +int main(int argc, char * argv[]) try +{ + std::string serial; + if (!device_with_streams({ RS2_STREAM_COLOR,RS2_STREAM_DEPTH }, serial)) + return EXIT_SUCCESS; + + // The following toggle is going to control + // if we will use CPU or GPU for depth data processing + bool use_gpu_processing = true; + + // Create and initialize GUI related objects + window app(1280, 720, "RealSense Align Example"); // Simple window handling + ImGui_ImplGlfw_Init(app, false); // ImGui library intializition + + // Once we have a window, initialize GL module + // Pass our window to enable sharing of textures between processed frames and the window + rs2::gl::init_processing(app, use_gpu_processing); + // Initialize rendering module: + rs2::gl::init_rendering(); + + rs2::gl::colorizer c; // Helper to colorize depth images + texture depth_image, color_image; // Helpers for renderig images + + // Create a pipeline to easily configure and start the camera + rs2::pipeline pipe; + rs2::config cfg; + if (!serial.empty()) + cfg.enable_device(serial); + cfg.enable_stream(RS2_STREAM_DEPTH); + cfg.enable_stream(RS2_STREAM_COLOR); + pipe.start(cfg); + + // Define two align objects. One will be used to align + // to depth viewport and the other to color. + // Creating align object is an expensive operation + // that should not be performed in the main loop + rs2::gl::align align_to_depth(RS2_STREAM_DEPTH); + rs2::gl::align align_to_color(RS2_STREAM_COLOR); + + float alpha = 0.5f; // Transparancy coefficient + direction dir = direction::to_depth; // Alignment direction + + while (app) // Application still alive? + { + // Using the align object, we block the application until a frameset is available + rs2::frameset frameset = pipe.wait_for_frames(); + + if (dir == direction::to_depth) + { + // Align all frames to depth viewport + frameset = align_to_depth.process(frameset); + } + else + { + // Align all frames to color viewport + frameset = align_to_color.process(frameset); + } + // With the aligned frameset we proceed as usual + auto depth = frameset.get_depth_frame(); + auto color = frameset.get_color_frame(); + + auto colorized_depth = c.colorize(depth); + + glEnable(GL_BLEND); + // Use the Alpha channel for blending + glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); + + if (dir == direction::to_depth) + { + // When aligning to depth, first render depth image + // and then overlay color on top with transparancy + depth_image.render(colorized_depth, { 0, 0, app.width(), app.height() }); + color_image.render(color, { 0, 0, app.width(), app.height() }, alpha); + } + else + { + // When aligning to color, first render color image + // and then overlay depth image on top + color_image.render(color, { 0, 0, app.width(), app.height() }); + depth_image.render(colorized_depth, { 0, 0, app.width(), app.height() }, 1 - alpha); + } + + glColor4f(1.f, 1.f, 1.f, 1.f); + glDisable(GL_BLEND); + + // Render the UI: + ImGui_ImplGlfw_NewFrame(1); + render_slider({ 15.f, app.height() - 60, app.width() - 30, app.height() }, &alpha, &dir); + ImGui::Render(); + } + + return EXIT_SUCCESS; +} +catch (const rs2::error & e) +{ + std::cerr << "RealSense error calling " << e.get_failed_function() << "(" << e.get_failed_args() << "):\n " << e.what() << std::endl; + return EXIT_FAILURE; +} +catch (const std::exception & e) +{ + std::cerr << e.what() << std::endl; + return EXIT_FAILURE; +} + +void render_slider(rect location, float* alpha, direction* dir) +{ + static const int flags = ImGuiWindowFlags_NoCollapse + | ImGuiWindowFlags_NoScrollbar + | ImGuiWindowFlags_NoSavedSettings + | ImGuiWindowFlags_NoTitleBar + | ImGuiWindowFlags_NoResize + | ImGuiWindowFlags_NoMove; + + ImGui::SetNextWindowPos({ location.x, location.y }); + ImGui::SetNextWindowSize({ location.w, location.h }); + + // Render transparency slider: + ImGui::Begin("slider", nullptr, flags); + ImGui::PushItemWidth(-1); + ImGui::SliderFloat("##Slider", alpha, 0.f, 1.f); + ImGui::PopItemWidth(); + if (ImGui::IsItemHovered()) + ImGui::SetTooltip("Texture Transparancy: %.3f", *alpha); + + // Render direction checkboxes: + bool to_depth = (*dir == direction::to_depth); + bool to_color = (*dir == direction::to_color); + + if (ImGui::Checkbox("Align To Depth", &to_depth)) + { + *dir = to_depth ? direction::to_depth : direction::to_color; + } + ImGui::SameLine(); + ImGui::SetCursorPosX(location.w - 140); + if (ImGui::Checkbox("Align To Color", &to_color)) + { + *dir = to_color ? direction::to_color : direction::to_depth; + } + + ImGui::End(); +} diff --git a/examples/readme.md b/examples/readme.md index a9e20d3736..e97a44abb0 100644 --- a/examples/readme.md +++ b/examples/readme.md @@ -31,6 +31,7 @@ For a detailed explanations and API documentation see our [Documentation](../doc |[DNN](../wrappers/openvino/dnn)| C++ & [OpenVINO](https://github.com/IntelRealSense/librealsense/tree/master/wrappers/openvino) | Intel RealSense camera used for real-time object-detection | :star::star: | [![Depth Sensing - Structured Light, Stereo and L500](https://img.shields.io/badge/-Depth-5bc3ff.svg)](./depth.md) | |[Software Device](./software-device)| C++ | Shows how to create a custom `rs2::device` | :star::star::star: | [![Depth Sensing - Structured Light, Stereo and L500](https://img.shields.io/badge/-Depth-5bc3ff.svg)](./depth.md) | |[GL](./gl)| C++ | Shows how to perform parts of frame processing using the GPU | :star::star::star: | [![Depth Sensing - Structured Light, Stereo and L500](https://img.shields.io/badge/-Depth-5bc3ff.svg)](./depth.md) | +|[GL Spatial Alignment](./align-gl)| C++ | Shows how to perform frame alignment (depth_to_color & color_to_depth) using the GPU | :star::star::star: | [![Depth Sensing - Structured Light, Stereo and L500](https://img.shields.io/badge/-Depth-5bc3ff.svg)](./depth.md) | |[Sensor Control](./sensor-control)| C++ | A tutorial for using the `rs2::sensor` API | :star::star::star: | [![Depth Sensing - Structured Light, Stereo and L500](https://img.shields.io/badge/-Depth-5bc3ff.svg)](./depth.md) | |[GrabCuts](../wrappers/opencv/grabcuts)| C++ & [OpenCV](https://github.com/IntelRealSense/librealsense/tree/master/wrappers/opencv#getting-started) | Simple background removal using the GrabCut algorithm | :star::star::star: | [![Depth Sensing - Structured Light, Stereo and L500](https://img.shields.io/badge/-Depth-5bc3ff.svg)](./depth.md) | |[Latency](../wrappers/opencv/latency-tool)| C++ & [OpenCV](https://github.com/IntelRealSense/librealsense/tree/master/wrappers/opencv#getting-started) | Basic latency estimation using computer vision | :star::star::star: | [![Depth Sensing - Structured Light, Stereo and L500](https://img.shields.io/badge/-Depth-5bc3ff.svg)](./depth.md) | diff --git a/src/gl/align-gl.cpp b/src/gl/align-gl.cpp index d9fef743d1..b8963665d0 100644 --- a/src/gl/align-gl.cpp +++ b/src/gl/align-gl.cpp @@ -138,7 +138,7 @@ void build_opengl_projection_for_intrinsics(matrix4& frustum, // additional row is inserted to map the z-coordinate to // OpenGL. matrix4 tproj; - tproj(0,0) = float(alpha); tproj(0,1) = float(skew); tproj(0,2) = 0.f; + tproj(0,0) = float(alpha); tproj(0,1) = float(skew); tproj(0,2) = float(u0); tproj(1,1) = float(beta); tproj(1,2) = float(v0); tproj(2,2) = float(-(N+F)); tproj(2,3) = float(-N*F); tproj(3,2) = 1.f;