diff --git a/README.md b/README.md index 9645903..c1bb180 100644 --- a/README.md +++ b/README.md @@ -44,11 +44,11 @@ This package is powered by [NVIDIA Isaac Transport for ROS (NITROS)](https://dev ## Performance -| Sample Graph

| Input Size

| AGX Orin

| Orin NX

| Orin Nano 8GB

| x86_64 w/ RTX 4060 Ti

| x86_64 w/ RTX 4090

| -|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| [SAM Image Segmentation Graph](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/benchmarks/isaac_ros_segment_anything_benchmark/scripts/isaac_ros_segment_anything_graph.py)


Full SAM

| 720p



| [2.22 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_sam_graph-agx_orin.json)


470 ms @ 30Hz

| –



| –



| –



| [14.6 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_sam_graph-x86_4090.json)


79 ms @ 30Hz

| -| [SAM Image Segmentation Graph](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/benchmarks/isaac_ros_segment_anything_benchmark/scripts/isaac_ros_mobile_segment_anything_graph.py)


Mobile SAM

| 720p



| [10.8 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_mobile_sam_graph-agx_orin.json)


880 ms @ 30Hz

| [5.13 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_mobile_sam_graph-orin_nx.json)


1500 ms @ 30Hz

| [2.22 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_mobile_sam_graph-orin_nano.json)


360 ms @ 30Hz

| [27.0 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_mobile_sam_graph-nuc_4060ti.json)


62 ms @ 30Hz

| [60.3 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_mobile_sam_graph-x86_4090.json)


27 ms @ 30Hz

| -| [TensorRT Graph](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/benchmarks/isaac_ros_unet_benchmark/scripts/isaac_ros_unet_graph.py)


PeopleSemSegNet

| 544p



| [371 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_unet_graph-agx_orin.json)


19 ms @ 30Hz

| [250 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_unet_graph-orin_nx.json)


20 ms @ 30Hz

| [163 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_unet_graph-orin_nano.json)


23 ms @ 30Hz

| [670 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_unet_graph-nuc_4060ti.json)


11 ms @ 30Hz

| [688 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_unet_graph-x86_4090.json)


9.3 ms @ 30Hz

| +| Sample Graph

| Input Size

| AGX Orin

| Orin NX

| Orin Nano 8GB

| x86_64 w/ RTX 4090

| +|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [SAM Image Segmentation Graph](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/benchmarks/isaac_ros_segment_anything_benchmark/scripts/isaac_ros_segment_anything_graph.py)


Full SAM

| 720p



| [2.22 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_sam_graph-agx_orin.json)


390 ms @ 30Hz

| –



| –



| [16.4 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_sam_graph-x86-4090.json)


280 ms @ 30Hz

| +| [SAM Image Segmentation Graph](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/benchmarks/isaac_ros_segment_anything_benchmark/scripts/isaac_ros_mobile_segment_anything_graph.py)


Mobile SAM

| 720p



| [8.75 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_mobile_sam_graph-agx_orin.json)


570 ms @ 30Hz

| [5.34 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_mobile_sam_graph-orin_nx.json)


1400 ms @ 30Hz

| [2.22 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_mobile_sam_graph-orin_nano.json)


340 ms @ 30Hz

| [68.6 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_mobile_sam_graph-x86-4090.json)


23 ms @ 30Hz

| +| [TensorRT Graph](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/benchmarks/isaac_ros_unet_benchmark/scripts/isaac_ros_unet_graph.py)


PeopleSemSegNet

| 544p



| [371 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_unet_graph-agx_orin.json)


19 ms @ 30Hz

| [250 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_unet_graph-orin_nx.json)


20 ms @ 30Hz

| [163 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_unet_graph-orin_nano.json)


23 ms @ 30Hz

| [670 fps](https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark/blob/main/results/isaac_ros_unet_graph-nuc_4060ti.json)


11 ms @ 30Hz

| --- @@ -78,4 +78,4 @@ Please visit the [Isaac ROS Documentation](https://nvidia-isaac-ros.github.io/re ## Latest -Update 2024-09-26: Update for ZED compatibility +Update 2024-12-10: Update to be compatible with JetPack 6.1 diff --git a/isaac_ros_gxf_extensions/gxf_isaac_ros_segment_anything/CMakeLists.txt b/isaac_ros_gxf_extensions/gxf_isaac_ros_segment_anything/CMakeLists.txt index 837cead..09df321 100644 --- a/isaac_ros_gxf_extensions/gxf_isaac_ros_segment_anything/CMakeLists.txt +++ b/isaac_ros_gxf_extensions/gxf_isaac_ros_segment_anything/CMakeLists.txt @@ -63,4 +63,10 @@ set_target_properties(${PROJECT_NAME} PROPERTIES # Install the binary file install(TARGETS ${PROJECT_NAME} DESTINATION share/${PROJECT_NAME}/gxf/lib) + +# Embed versioning information into installed files +ament_index_get_resource(ISAAC_ROS_COMMON_CMAKE_PATH isaac_ros_common_cmake_path isaac_ros_common) +include("${ISAAC_ROS_COMMON_CMAKE_PATH}/isaac_ros_common-version-info.cmake") +generate_version_info(${PROJECT_NAME}) + ament_auto_package(INSTALL_TO_SHARE) \ No newline at end of file diff --git a/isaac_ros_gxf_extensions/gxf_isaac_ros_segment_anything/package.xml b/isaac_ros_gxf_extensions/gxf_isaac_ros_segment_anything/package.xml index e01ccb2..1dbdfd5 100644 --- a/isaac_ros_gxf_extensions/gxf_isaac_ros_segment_anything/package.xml +++ b/isaac_ros_gxf_extensions/gxf_isaac_ros_segment_anything/package.xml @@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0 gxf_isaac_ros_segment_anything - 3.1.0 + 3.2.0 Segmentation Anything extension. Isaac ROS Maintainers diff --git a/isaac_ros_gxf_extensions/gxf_isaac_ros_unet/CMakeLists.txt b/isaac_ros_gxf_extensions/gxf_isaac_ros_unet/CMakeLists.txt index e77b93f..e916707 100644 --- a/isaac_ros_gxf_extensions/gxf_isaac_ros_unet/CMakeLists.txt +++ b/isaac_ros_gxf_extensions/gxf_isaac_ros_unet/CMakeLists.txt @@ -70,5 +70,11 @@ set_target_properties(${PROJECT_NAME} PROPERTIES # Install the binary file install(TARGETS ${PROJECT_NAME} DESTINATION share/${PROJECT_NAME}/gxf/lib) + +# Embed versioning information into installed files +ament_index_get_resource(ISAAC_ROS_COMMON_CMAKE_PATH isaac_ros_common_cmake_path isaac_ros_common) +include("${ISAAC_ROS_COMMON_CMAKE_PATH}/isaac_ros_common-version-info.cmake") +generate_version_info(${PROJECT_NAME}) + ament_auto_package(INSTALL_TO_SHARE) diff --git a/isaac_ros_gxf_extensions/gxf_isaac_ros_unet/package.xml b/isaac_ros_gxf_extensions/gxf_isaac_ros_unet/package.xml index 43e76a3..354a04d 100644 --- a/isaac_ros_gxf_extensions/gxf_isaac_ros_unet/package.xml +++ b/isaac_ros_gxf_extensions/gxf_isaac_ros_unet/package.xml @@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0 gxf_isaac_ros_unet - 3.1.0 + 3.2.0 Segmentation post-processor extension. Isaac ROS Maintainers diff --git a/isaac_ros_peoplesemseg_models_install/CMakeLists.txt b/isaac_ros_peoplesemseg_models_install/CMakeLists.txt index f200387..8cc1cb8 100644 --- a/isaac_ros_peoplesemseg_models_install/CMakeLists.txt +++ b/isaac_ros_peoplesemseg_models_install/CMakeLists.txt @@ -29,4 +29,10 @@ install_isaac_ros_asset(install_peoplesemsegnet_shuffleseg) install(PROGRAMS asset_scripts/install_peoplesemsegnet_shuffleseg.sh DESTINATION lib/${PROJECT_NAME}) install(PROGRAMS asset_scripts/install_peoplesemsegnet_vanilla.sh DESTINATION lib/${PROJECT_NAME}) + +# Embed versioning information into installed files +ament_index_get_resource(ISAAC_ROS_COMMON_CMAKE_PATH isaac_ros_common_cmake_path isaac_ros_common) +include("${ISAAC_ROS_COMMON_CMAKE_PATH}/isaac_ros_common-version-info.cmake") +generate_version_info(${PROJECT_NAME}) + ament_auto_package(INSTALL_TO_SHARE) diff --git a/isaac_ros_peoplesemseg_models_install/asset_scripts/install_peoplesemsegnet_shuffleseg.sh b/isaac_ros_peoplesemseg_models_install/asset_scripts/install_peoplesemsegnet_shuffleseg.sh index b0ad66e..bd755d1 100755 --- a/isaac_ros_peoplesemseg_models_install/asset_scripts/install_peoplesemsegnet_shuffleseg.sh +++ b/isaac_ros_peoplesemseg_models_install/asset_scripts/install_peoplesemsegnet_shuffleseg.sh @@ -7,17 +7,17 @@ # distribution of this software and related documentation without an express # license agreement from NVIDIA CORPORATION is strictly prohibited. -# Download and tao-convert ESS models. +# Download and TRT-compile PeopleSemSeg-ShuffleSeg models. # * Models will be stored in the isaac_ros_assets dir # * The script must be called with the --eula argument prior to downloading. set -e -ASSET_NAME="deployable_shuffleseg_unet_amr_v1.0" -EULA_URL="https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplesemsegnet_amr" +ASSET_NAME="optimized_deployable_shuffleseg_unet_amr_v1.0" +EULA_URL="https://catalog.ngc.nvidia.com/orgs/nvidia/teams/isaac/models/optimized-peoplesemseg-amr" ASSET_DIR="${ISAAC_ROS_WS}/isaac_ros_assets/models/peoplesemsegnet/${ASSET_NAME}" ASSET_INSTALL_PATHS="${ASSET_DIR}/1/model.plan" -MODEL_URL="https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplesemsegnet_amr/deployable_v1.1/files?redirect=true&path=peoplesemsegnet_amr_rel.onnx" +MODEL_URL="https://api.ngc.nvidia.com/v2/models/org/nvidia/team/isaac/optimized-peoplesemseg-amr/v1.0/files?redirect=true&path=model.onnx" source "isaac_ros_asset_eula.sh" @@ -27,9 +27,9 @@ wget "${MODEL_URL}" -O "${ASSET_DIR}/model.onnx" echo "Converting PeopleSemSegnet shuffleseg amr onnx file to plan file." /usr/src/tensorrt/bin/trtexec \ - --maxShapes="input_2:0":1x3x544x960 \ - --minShapes="input_2:0":1x3x544x960 \ - --optShapes="input_2:0":1x3x544x960 \ + --maxShapes="input_2":1x544x960x3 \ + --minShapes="input_2":1x544x960x3 \ + --optShapes="input_2":1x544x960x3 \ --fp16 \ --saveEngine="${ASSET_INSTALL_PATHS}" \ --onnx="${ASSET_DIR}/model.onnx" @@ -41,9 +41,9 @@ platform: "tensorrt_plan" max_batch_size: 0 input [ { - name: "input_2:0" + name: "input_2" data_type: TYPE_FP32 - dims: [ 1, 3, 544, 960 ] + dims: [ 1, 544, 960, 3 ] } ] output [ diff --git a/isaac_ros_peoplesemseg_models_install/asset_scripts/install_peoplesemsegnet_vanilla.sh b/isaac_ros_peoplesemseg_models_install/asset_scripts/install_peoplesemsegnet_vanilla.sh index 5fdb3ef..935efa6 100755 --- a/isaac_ros_peoplesemseg_models_install/asset_scripts/install_peoplesemsegnet_vanilla.sh +++ b/isaac_ros_peoplesemseg_models_install/asset_scripts/install_peoplesemsegnet_vanilla.sh @@ -7,33 +7,33 @@ # distribution of this software and related documentation without an express # license agreement from NVIDIA CORPORATION is strictly prohibited. -# Download and tao-convert ESS models. +# Download and TRT-compile PeopleSemSeg-Vanilla models. # * Models will be stored in the isaac_ros_assets dir # * The script must be called with the --eula argument prior to downloading. set -e -ASSET_NAME="deployable_quantized_vanilla_unet_v2.0" +ASSET_NAME="deployable_quantized_vanilla_unet_onnx_v2.0" EULA_URL="https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tao/models/peoplesemsegnet" ASSET_DIR="${ISAAC_ROS_WS}/isaac_ros_assets/models/peoplesemsegnet/${ASSET_NAME}" ASSET_INSTALL_PATHS="${ASSET_DIR}/1/model.plan" -MODEL_URL="https://api.ngc.nvidia.com/v2/models/nvidia/tao/peoplesemsegnet/versions/deployable_quantized_vanilla_unet_v2.0/files/peoplesemsegnet_vanilla_unet_dynamic_etlt_int8_fp16.etlt" +MODEL_URL="https://api.ngc.nvidia.com/v2/models/org/nvidia/team/tao/peoplesemsegnet/deployable_quantized_vanilla_unet_onnx_v2.0/files?redirect=true&path=peoplesemsegnet_vanilla_unet_dynamic_etlt_int8_fp16.onnx" source "isaac_ros_asset_eula.sh" mkdir -p $(dirname "$ASSET_INSTALL_PATHS") -wget "${MODEL_URL}" -O "${ASSET_DIR}/model.etlt" +wget "${MODEL_URL}" -O "${ASSET_DIR}/model.onnx" -echo "Converting PeopleSemSegnet etlt file to plan file." -/opt/nvidia/tao/tao-converter \ - -k tlt_encode \ - -d 3,544,960 \ - -p input_1:0,1x3x544x960,1x3x544x960,1x3x544x960 \ - -t fp16 \ - -e "${ASSET_INSTALL_PATHS}" \ - -o argmax_1 \ - "${ASSET_DIR}/model.etlt" +echo "Converting PeopleSemSegnet onnx file to plan file." +/usr/src/tensorrt/bin/trtexec \ + --maxShapes="input_1:0":1x3x544x960 \ + --minShapes="input_1:0":1x3x544x960 \ + --optShapes="input_1:0":1x3x544x960 \ + --onnx="${ASSET_DIR}/model.onnx" \ + --saveEngine="${ASSET_DIR}/1/model.plan" \ + --fp16 \ + --skipInference # Create config.pbtxt config_file_text=$( diff --git a/isaac_ros_peoplesemseg_models_install/package.xml b/isaac_ros_peoplesemseg_models_install/package.xml index 898bb1c..68122d8 100644 --- a/isaac_ros_peoplesemseg_models_install/package.xml +++ b/isaac_ros_peoplesemseg_models_install/package.xml @@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0 isaac_ros_peoplesemseg_models_install - 3.1.0 + 3.2.0 Scripts for installing people segmentation models Isaac ROS Maintainers diff --git a/isaac_ros_segformer/CMakeLists.txt b/isaac_ros_segformer/CMakeLists.txt index b7f65c7..1b43881 100644 --- a/isaac_ros_segformer/CMakeLists.txt +++ b/isaac_ros_segformer/CMakeLists.txt @@ -40,4 +40,10 @@ if(BUILD_TESTING) add_launch_test(test/isaac_ros_segformer_pol_test.py TIMEOUT "300") endif() + +# Embed versioning information into installed files +ament_index_get_resource(ISAAC_ROS_COMMON_CMAKE_PATH isaac_ros_common_cmake_path isaac_ros_common) +include("${ISAAC_ROS_COMMON_CMAKE_PATH}/isaac_ros_common-version-info.cmake") +generate_version_info(${PROJECT_NAME}) + ament_auto_package(INSTALL_TO_SHARE launch) diff --git a/isaac_ros_segformer/package.xml b/isaac_ros_segformer/package.xml index 643be43..da665d0 100644 --- a/isaac_ros_segformer/package.xml +++ b/isaac_ros_segformer/package.xml @@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0 isaac_ros_segformer - 3.1.0 + 3.2.0 Segformer model processing Isaac ROS Maintainers diff --git a/isaac_ros_segformer/test/dummy_model/.gitignore b/isaac_ros_segformer/test/dummy_model/.gitignore new file mode 100644 index 0000000..00afe2c --- /dev/null +++ b/isaac_ros_segformer/test/dummy_model/.gitignore @@ -0,0 +1,2 @@ +*.plan +*.onnx \ No newline at end of file diff --git a/isaac_ros_segformer/test/dummy_model/.gitkeep b/isaac_ros_segformer/test/dummy_model/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/isaac_ros_segformer/test/dummy_model/model.dummy.onnx b/isaac_ros_segformer/test/dummy_model/model.dummy.onnx deleted file mode 100644 index b25d188..0000000 --- a/isaac_ros_segformer/test/dummy_model/model.dummy.onnx +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b9163a8da58d83009183ac7a8089a382b3b53ee3c393e8f7f97d9b89fe2de1c1 -size 391 diff --git a/isaac_ros_segformer/test/isaac_ros_segformer_pol_test.py b/isaac_ros_segformer/test/isaac_ros_segformer_pol_test.py index d309351..1207842 100644 --- a/isaac_ros_segformer/test/isaac_ros_segformer_pol_test.py +++ b/isaac_ros_segformer/test/isaac_ros_segformer_pol_test.py @@ -32,20 +32,18 @@ import time from ament_index_python.packages import get_package_share_directory -from isaac_ros_test import IsaacROSBaseTest, JSONConversion +from isaac_ros_test import IsaacROSBaseTest, JSONConversion, MockModelGenerator import launch from launch.actions import IncludeLaunchDescription from launch.launch_description_sources import PythonLaunchDescriptionSource from launch_ros.actions.composable_node_container import ComposableNodeContainer from launch_ros.descriptions.composable_node import ComposableNode import launch_testing - import numpy as np - import pytest import rclpy - from sensor_msgs.msg import CameraInfo, Image +import torch _TEST_CASE_NAMESPACE = 'segformer_node_test' @@ -74,6 +72,17 @@ def generate_test_description(): if e.errno != errno.ENOENT: print('File exists but error deleting /tmp/trt_engine.plan') + # Generate a mock model with SegFormer-like I/O + MockModelGenerator.generate( + input_bindings=[ + MockModelGenerator.Binding('input', [-1, 3, 512, 512], torch.float32) + ], + output_bindings=[ + MockModelGenerator.Binding('output', [-1, 1, 512, 512], torch.int64), + ], + output_onnx_path=model_file_path + ) + encoder_dir = get_package_share_directory('isaac_ros_dnn_image_encoder') encoder_node_launch = IncludeLaunchDescription( PythonLaunchDescriptionSource( diff --git a/isaac_ros_segment_anything/CMakeLists.txt b/isaac_ros_segment_anything/CMakeLists.txt index 61e9d9d..1c96e6b 100644 --- a/isaac_ros_segment_anything/CMakeLists.txt +++ b/isaac_ros_segment_anything/CMakeLists.txt @@ -79,4 +79,10 @@ if(BUILD_TESTING) add_launch_test(test/isaac_ros_segment_anything_point_prompt_pol.py TIMEOUT "300") endif() + +# Embed versioning information into installed files +ament_index_get_resource(ISAAC_ROS_COMMON_CMAKE_PATH isaac_ros_common_cmake_path isaac_ros_common) +include("${ISAAC_ROS_COMMON_CMAKE_PATH}/isaac_ros_common-version-info.cmake") +generate_version_info(${PROJECT_NAME}) + ament_auto_package(INSTALL_TO_SHARE config launch) diff --git a/isaac_ros_segment_anything/config/segment_anything_data_encoder_node.yaml b/isaac_ros_segment_anything/config/segment_anything_data_encoder_node.yaml index 9e67eac..ce557de 100644 --- a/isaac_ros_segment_anything/config/segment_anything_data_encoder_node.yaml +++ b/isaac_ros_segment_anything/config/segment_anything_data_encoder_node.yaml @@ -21,7 +21,7 @@ components: - name: image_receiver type: nvidia::gxf::DoubleBufferReceiver parameters: - capacity: 12 + capacity: 2 - type: nvidia::gxf::MessageAvailableSchedulingTerm parameters: receiver: image_receiver @@ -29,7 +29,7 @@ components: - name: prompt_receiver type: nvidia::gxf::DoubleBufferReceiver parameters: - capacity: 12 + capacity: 2 - type: nvidia::gxf::MessageAvailableSchedulingTerm parameters: receiver: prompt_receiver @@ -37,7 +37,7 @@ components: - name: mask_receiver type: nvidia::gxf::DoubleBufferReceiver parameters: - capacity: 12 + capacity: 1 - type: nvidia::gxf::MessageAvailableSchedulingTerm parameters: receiver: mask_receiver @@ -45,7 +45,7 @@ components: - name: image_transmitter type: nvidia::gxf::DoubleBufferTransmitter parameters: - capacity: 12 + capacity: 1 - type: nvidia::gxf::DownstreamReceptiveSchedulingTerm parameters: transmitter: image_transmitter @@ -53,7 +53,7 @@ components: - name: prompt_transmitter type: nvidia::gxf::DoubleBufferTransmitter parameters: - capacity: 12 + capacity: 1 - type: nvidia::gxf::DownstreamReceptiveSchedulingTerm parameters: transmitter: prompt_transmitter @@ -61,7 +61,7 @@ components: - name: mask_transmitter type: nvidia::gxf::DoubleBufferTransmitter parameters: - capacity: 12 + capacity: 1 - type: nvidia::gxf::DownstreamReceptiveSchedulingTerm parameters: transmitter: mask_transmitter @@ -76,7 +76,7 @@ components: - name: input_prompt type: nvidia::gxf::DoubleBufferReceiver parameters: - capacity: 12 + capacity: 1 - type: nvidia::gxf::MessageAvailableSchedulingTerm parameters: receiver: input_prompt @@ -84,7 +84,7 @@ components: - name: output_buffer type: nvidia::gxf::DoubleBufferTransmitter parameters: - capacity: 12 + capacity: 1 - type: nvidia::gxf::DownstreamReceptiveSchedulingTerm parameters: transmitter: output_buffer @@ -110,7 +110,7 @@ components: - name: input_prompt type: nvidia::gxf::DoubleBufferReceiver parameters: - capacity: 12 + capacity: 1 - type: nvidia::gxf::MessageAvailableSchedulingTerm parameters: receiver: input_prompt @@ -118,7 +118,7 @@ components: - name: input_image type: nvidia::gxf::DoubleBufferReceiver parameters: - capacity: 12 + capacity: 1 - type: nvidia::gxf::MessageAvailableSchedulingTerm parameters: receiver: input_image @@ -126,7 +126,7 @@ components: - name: input_mask type: nvidia::gxf::DoubleBufferReceiver parameters: - capacity: 12 + capacity: 1 - type: nvidia::gxf::MessageAvailableSchedulingTerm parameters: receiver: input_mask @@ -134,7 +134,7 @@ components: - name: output_buffer type: nvidia::gxf::DoubleBufferTransmitter parameters: - capacity: 12 + capacity: 1 - type: nvidia::gxf::DownstreamReceptiveSchedulingTerm parameters: transmitter: output_buffer diff --git a/isaac_ros_segment_anything/config/segment_anything_decoder_node.yaml b/isaac_ros_segment_anything/config/segment_anything_decoder_node.yaml index c1d3977..4087fd5 100644 --- a/isaac_ros_segment_anything/config/segment_anything_decoder_node.yaml +++ b/isaac_ros_segment_anything/config/segment_anything_decoder_node.yaml @@ -21,7 +21,7 @@ components: - name: input_tensor type: nvidia::gxf::DoubleBufferReceiver parameters: - capacity: 12 + capacity: 1 - type: nvidia::gxf::MessageAvailableSchedulingTerm parameters: receiver: input_tensor @@ -29,7 +29,7 @@ components: - name: output_buffer type: nvidia::gxf::DoubleBufferTransmitter parameters: - capacity: 12 + capacity: 1 - type: nvidia::gxf::DownstreamReceptiveSchedulingTerm parameters: transmitter: output_buffer @@ -64,7 +64,7 @@ components: - name: input_raw_segmentation_mask type: nvidia::gxf::DoubleBufferReceiver parameters: - capacity: 2 + capacity: 1 - type: nvidia::gxf::MessageAvailableSchedulingTerm parameters: receiver: input_raw_segmentation_mask diff --git a/isaac_ros_segment_anything/launch/isaac_ros_segment_anything_core.launch.py b/isaac_ros_segment_anything/launch/isaac_ros_segment_anything_core.launch.py index df6e5d9..166461d 100644 --- a/isaac_ros_segment_anything/launch/isaac_ros_segment_anything_core.launch.py +++ b/isaac_ros_segment_anything/launch/isaac_ros_segment_anything_core.launch.py @@ -50,6 +50,7 @@ def get_composable_nodes(interface_specs: Dict[str, Any]) -> Dict[str, Composabl has_input_mask = LaunchConfiguration('has_input_mask') orig_img_dims = [interface_specs['input_image']['height'], interface_specs['input_image']['width']] + input_image_encoding = LaunchConfiguration('input_image_encoding') return { @@ -62,6 +63,7 @@ def get_composable_nodes(interface_specs: Dict[str, Any]) -> Dict[str, Composabl 'output_height': SAM_MODEL_INPUT_SIZE, 'keep_aspect_ratio': True, 'disable_padding': True, + 'encoding_desired': input_image_encoding, 'input_width': interface_specs['input_image']['width'], 'input_height': interface_specs['input_image']['height'] }], @@ -265,6 +267,11 @@ def get_launch_actions(interface_specs: Dict[str, Any]) -> \ 'color_segmentation_mask_encoding', default_value='rgb8', description='The image encoding of the colored segmentation mask (rgb8 or bgr8)'), + 'input_image_encoding': DeclareLaunchArgument( + 'input_image_encoding', + default_value='rgb8', + description='The image encoding of the input image \ + (rgb8 or bgr8 or bgra8 or rgba8)'), 'prompt_input_type': DeclareLaunchArgument( 'prompt_input_type', default_value='bbox', diff --git a/isaac_ros_segment_anything/package.xml b/isaac_ros_segment_anything/package.xml index accaea4..619a62f 100644 --- a/isaac_ros_segment_anything/package.xml +++ b/isaac_ros_segment_anything/package.xml @@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0 isaac_ros_segment_anything - 3.1.0 + 3.2.0 Segment Anything model processing Isaac ROS Maintainers diff --git a/isaac_ros_segment_anything/scripts/visualize_mask.py b/isaac_ros_segment_anything/scripts/visualize_mask.py index 7472b19..b9cc34f 100755 --- a/isaac_ros_segment_anything/scripts/visualize_mask.py +++ b/isaac_ros_segment_anything/scripts/visualize_mask.py @@ -16,9 +16,10 @@ # limitations under the License. # # SPDX-License-Identifier: Apache-2.0 - +import cv2 import cv_bridge from isaac_ros_tensor_list_interfaces.msg import TensorList +from message_filters import Subscriber, TimeSynchronizer import numpy as np import rclpy from rclpy.node import Node @@ -29,9 +30,13 @@ class SegmentAnythingVisualization(Node): def __init__(self): super().__init__('segment_anything_visualizer') + self._bridge = cv_bridge.CvBridge() - self._mask_subscriber = self.create_subscription( - TensorList, 'segment_anything/raw_segmentation_mask', self.callback, 10) + self._mask_subscriber = Subscriber( + self, TensorList, 'segment_anything/raw_segmentation_mask') + + self._image_subscriber = Subscriber( + self, Image, '/yolov8_encoder/resize/image') self._processed_image_pub = self.create_publisher( Image, 'segment_anything/colored_segmentation_mask', 10) @@ -40,8 +45,11 @@ def __init__(self): 'FF8C00', 'FFD700', '00FF00', 'BA55D3', '00FA9A', '00FFFF', '0000FF', 'F08080', 'FF00FF', '1E90FF', 'DDA0DD', 'FF1493', '87CEFA', 'FFDEAD'] + self.sync = TimeSynchronizer([self._image_subscriber, self._mask_subscriber], 50) + self.sync.registerCallback(self.callback) - def callback(self, masks): + def callback(self, img, masks): + input_image = self._bridge.imgmsg_to_cv2(img, 'rgb8') tensor = masks.tensors[0] shape = tensor.shape @@ -55,14 +63,15 @@ def callback(self, masks): for n in range(dimensions[0]): palette_idx = n - if(n >= len(self._color_palette)): + if (n >= len(self._color_palette)): palette_idx = len(self._color_palette) - 1 rgb_val = [int(self._color_palette[palette_idx][i:i+2], 16) for i in (0, 2, 4)] rgb_image[:, :, :][np.where(data[n, 0, :, :] > 0)] = np.array(rgb_val) + result_image = cv2.addWeighted(input_image, 0.5, rgb_image, 0.5, 0) processed_img = self._bridge.cv2_to_imgmsg( - rgb_image, encoding='rgb8') + result_image, encoding='rgb8') self._processed_image_pub.publish(processed_img) diff --git a/isaac_ros_segment_anything/test/segment_anything_data_encoder_node_test.cpp b/isaac_ros_segment_anything/test/segment_anything_data_encoder_node_test.cpp index 880e25d..5563c1c 100644 --- a/isaac_ros_segment_anything/test/segment_anything_data_encoder_node_test.cpp +++ b/isaac_ros_segment_anything/test/segment_anything_data_encoder_node_test.cpp @@ -22,15 +22,12 @@ // Objective: to cover code lines where exceptions are thrown // Approach: send Invalid Arguments for node parameters to trigger the exception + TEST(segment_anything_data_encoder_node_test, test_invalid_input_prompt_type) { rclcpp::init(0, nullptr); rclcpp::NodeOptions options; - options.arguments( - { - "--ros-args", - "-p", "prompt_input_type:=''", - }); + options.append_parameter_override("prompt_input_type", ""); EXPECT_THROW( { try { @@ -39,6 +36,9 @@ TEST(segment_anything_data_encoder_node_test, test_invalid_input_prompt_type) } catch (const std::invalid_argument & e) { EXPECT_THAT(e.what(), testing::HasSubstr("Received invalid input prompt type")); throw; + } catch (const rclcpp::exceptions::InvalidParameterValueException & e) { + EXPECT_THAT(e.what(), testing::HasSubstr("No parameter value set")); + throw; } }, std::invalid_argument); rclcpp::shutdown(); diff --git a/isaac_ros_unet/CMakeLists.txt b/isaac_ros_unet/CMakeLists.txt index 893f64b..2eddc5b 100644 --- a/isaac_ros_unet/CMakeLists.txt +++ b/isaac_ros_unet/CMakeLists.txt @@ -39,6 +39,13 @@ if(BUILD_TESTING) find_package(ament_lint_auto REQUIRED) ament_lint_auto_find_test_dependencies() + # Gtest for unet decoder node + ament_add_gtest(unet_decoder_node_test test/unet_decoder_node_test.cpp) + target_link_libraries(unet_decoder_node_test unet_decoder_node) + target_include_directories(unet_decoder_node_test PUBLIC include/isaac_ros_unet/) + target_include_directories(unet_decoder_node_test PUBLIC /usr/src/googletest/googlemock/include/) + ament_target_dependencies(unet_decoder_node_test rclcpp) + ament_target_dependencies(unet_decoder_node_test isaac_ros_nitros) # The FindPythonInterp and FindPythonLibs modules are removed if(POLICY CMP0148) @@ -49,4 +56,10 @@ if(BUILD_TESTING) add_launch_test(test/isaac_ros_unet_pol_test.py TIMEOUT "600") endif() + +# Embed versioning information into installed files +ament_index_get_resource(ISAAC_ROS_COMMON_CMAKE_PATH isaac_ros_common_cmake_path isaac_ros_common) +include("${ISAAC_ROS_COMMON_CMAKE_PATH}/isaac_ros_common-version-info.cmake") +generate_version_info(${PROJECT_NAME}) + ament_auto_package(INSTALL_TO_SHARE config launch) diff --git a/isaac_ros_unet/launch/isaac_ros_argus_unet_triton.launch.py b/isaac_ros_unet/launch/isaac_ros_argus_unet_triton.launch.py index 363878b..4b6d990 100644 --- a/isaac_ros_unet/launch/isaac_ros_argus_unet_triton.launch.py +++ b/isaac_ros_unet/launch/isaac_ros_argus_unet_triton.launch.py @@ -45,6 +45,10 @@ def generate_launch_description(): 'encoder_image_stddev', default_value='[0.5, 0.5, 0.5]', description='The standard deviation for image normalization'), + DeclareLaunchArgument( + 'use_planar_input', + default_value='True', + description='Whether the input image should be in planar format or not'), DeclareLaunchArgument( 'model_name', default_value='', @@ -104,6 +108,7 @@ def generate_launch_description(): network_image_height = LaunchConfiguration('network_image_height') encoder_image_mean = LaunchConfiguration('encoder_image_mean') encoder_image_stddev = LaunchConfiguration('encoder_image_stddev') + use_planar_input = LaunchConfiguration('use_planar_input') # Triton parameters model_name = LaunchConfiguration('model_name') @@ -143,10 +148,10 @@ def generate_launch_description(): ) # Parameters preconfigured for PeopleSemSegNet. - encoder_dir = get_package_share_directory('isaac_ros_dnn_image_encoder') + encoder_dir = get_package_share_directory('isaac_ros_unet') encoder_node_launch = IncludeLaunchDescription( PythonLaunchDescriptionSource( - [os.path.join(encoder_dir, 'launch', 'dnn_image_encoder.launch.py')] + [os.path.join(encoder_dir, 'launch', 'isaac_ros_unet_encoder.launch.py')] ), launch_arguments={ 'input_image_width': str(1920), @@ -156,6 +161,7 @@ def generate_launch_description(): 'image_mean': encoder_image_mean, 'image_stddev': encoder_image_stddev, 'enable_padding': 'True', + 'use_planar_input': use_planar_input, 'image_input_topic': '/image_rect', 'camera_info_input_topic': '/camera_info_rect', 'tensor_output_topic': '/tensor_pub', diff --git a/isaac_ros_unet/launch/isaac_ros_unet_core.launch.py b/isaac_ros_unet/launch/isaac_ros_unet_core.launch.py index 9fb5a42..c7053b9 100644 --- a/isaac_ros_unet/launch/isaac_ros_unet_core.launch.py +++ b/isaac_ros_unet/launch/isaac_ros_unet_core.launch.py @@ -49,6 +49,9 @@ def get_composable_nodes(interface_specs: Dict[str, Any]) -> Dict[str, Composabl mask_width = LaunchConfiguration('mask_width') mask_height = LaunchConfiguration('mask_height') + # Alpha Blend parameters + alpha = LaunchConfiguration('alpha') + return { 'unet_inference_node': ComposableNode( name='unet_inference', @@ -81,6 +84,22 @@ def get_composable_nodes(interface_specs: Dict[str, Any]) -> Dict[str, Composabl 0x00FA9A, 0x00FFFF, 0x0000FF, 0xF08080, 0xFF00FF, 0x1E90FF, 0xDDA0DD, 0xFF1493, 0x87CEFA, 0xFFDEAD], }], + ), + 'alpha_blend_node': ComposableNode( + name='alpha_blend', + package='isaac_ros_image_proc', + plugin='nvidia::isaac_ros::image_proc::AlphaBlendNode', + parameters=[{ + 'alpha': alpha, + 'mask_queue_size': 50, + 'image_queue_size': 50, + 'sync_queue_size': 50, + }], + remappings=[ + ('mask_input', '/unet/colored_segmentation_mask'), + ('image_input', '/unet_encoder/converted/image'), + ('blended_image', '/segmentation_image_overlay') + ], ) } @@ -89,14 +108,19 @@ def get_launch_actions(interface_specs: Dict[str, Any]) -> \ Dict[str, launch.actions.OpaqueFunction]: # DNN Image Encoder parameters + input_qos = LaunchConfiguration('input_qos') network_image_width = LaunchConfiguration('network_image_width') network_image_height = LaunchConfiguration('network_image_height') encoder_image_mean = LaunchConfiguration('encoder_image_mean') encoder_image_stddev = LaunchConfiguration('encoder_image_stddev') + use_planar_input = LaunchConfiguration('use_planar_input') - encoder_dir = get_package_share_directory('isaac_ros_dnn_image_encoder') - + encoder_dir = get_package_share_directory('isaac_ros_unet') return { + 'input_qos': DeclareLaunchArgument( + 'input_qos', + default_value='DEFAULT', + description='The QoS profile of the resize node subscriber'), 'network_image_width': DeclareLaunchArgument( 'network_image_width', default_value='960', @@ -113,6 +137,10 @@ def get_launch_actions(interface_specs: Dict[str, Any]) -> \ 'encoder_image_stddev', default_value='[0.5, 0.5, 0.5]', description='The standard deviation for image normalization'), + 'use_planar_input': DeclareLaunchArgument( + 'use_planar_input', + default_value='True', + description='Whether the input image should be in planar format or not'), 'model_file_path': DeclareLaunchArgument( 'model_file_path', default_value='', @@ -176,11 +204,17 @@ def get_launch_actions(interface_specs: Dict[str, Any]) -> \ 'mask_height', default_value='544', description='The height of the segmentation mask'), + 'alpha': DeclareLaunchArgument( + 'alpha', + default_value='0.5', + description='The alpha value for alpha blending.', + ), 'dope_encoder_launch': IncludeLaunchDescription( PythonLaunchDescriptionSource( - [os.path.join(encoder_dir, 'launch', 'dnn_image_encoder.launch.py')] + [os.path.join(encoder_dir, 'launch', 'isaac_ros_unet_encoder.launch.py')] ), launch_arguments={ + 'input_qos': input_qos, 'input_image_width': str(interface_specs['camera_resolution']['width']), 'input_image_height': str(interface_specs['camera_resolution']['height']), 'network_image_width': network_image_width, @@ -188,6 +222,7 @@ def get_launch_actions(interface_specs: Dict[str, Any]) -> \ 'image_mean': encoder_image_mean, 'image_stddev': encoder_image_stddev, 'enable_padding': 'True', + 'use_planar_input': use_planar_input, 'attach_to_shared_component_container': 'True', 'component_container_name': '/isaac_ros_examples/container', 'dnn_image_encoder_namespace': 'unet_encoder', diff --git a/isaac_ros_unet/launch/isaac_ros_unet_encoder.launch.py b/isaac_ros_unet/launch/isaac_ros_unet_encoder.launch.py new file mode 100644 index 0000000..c62c861 --- /dev/null +++ b/isaac_ros_unet/launch/isaac_ros_unet_encoder.launch.py @@ -0,0 +1,338 @@ +# SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +# Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +import ast + +from launch import LaunchDescription +from launch.actions import (DeclareLaunchArgument, GroupAction, OpaqueFunction) +from launch.conditions import UnlessCondition +from launch.substitutions import LaunchConfiguration +from launch_ros.actions import LoadComposableNodes, Node, PushRosNamespace +from launch_ros.descriptions import ComposableNode + + +def launch_setup(context, *args, **kwargs): + input_image_width = int( + context.perform_substitution(LaunchConfiguration('input_image_width')) + ) + input_image_height = int( + context.perform_substitution(LaunchConfiguration('input_image_height')) + ) + + network_image_width = int( + context.perform_substitution(LaunchConfiguration('network_image_width')) + ) + network_image_height = int( + context.perform_substitution(LaunchConfiguration('network_image_height')) + ) + enable_padding = ast.literal_eval( + context.perform_substitution(LaunchConfiguration('enable_padding')) + ) + use_planar_input = ast.literal_eval( + context.perform_substitution(LaunchConfiguration('use_planar_input')) + ) + + input_qos = LaunchConfiguration('input_qos') + output_qos = LaunchConfiguration('output_qos') + + keep_aspect_ratio = LaunchConfiguration('keep_aspect_ratio') + crop_mode = LaunchConfiguration('crop_mode') + + encoding_desired = LaunchConfiguration('encoding_desired') + input_encoding = LaunchConfiguration('input_encoding') + final_tensor_name = LaunchConfiguration('final_tensor_name') + + image_mean = LaunchConfiguration('image_mean') + image_stddev = LaunchConfiguration('image_stddev') + num_blocks = LaunchConfiguration('num_blocks') + + image_input_topic = LaunchConfiguration('image_input_topic', default='image') + camera_info_input_topic = LaunchConfiguration('camera_info_input_topic', default='camera_info') + tensor_output_topic = LaunchConfiguration('tensor_output_topic', default='encoded_tensor') + + attach_to_shared_component_container_arg = LaunchConfiguration( + 'attach_to_shared_component_container', default=False + ) + component_container_name_arg = LaunchConfiguration( + 'component_container_name', default='dnn_image_encoder_container' + ) + dnn_image_encoder_namespace = LaunchConfiguration('dnn_image_encoder_namespace') + + resize_factor = 1.0 + if not enable_padding: + width_scalar = input_image_width / network_image_width + height_scalar = input_image_height / network_image_height + if width_scalar != height_scalar: + resize_factor = min(width_scalar, height_scalar) + + resize_image_width = int(network_image_width * resize_factor) + resize_image_height = int(network_image_height * resize_factor) + + # If we do not attach to a shared component container we have to create our own container. + dnn_image_encoder_container = Node( + name=component_container_name_arg, + package='rclcpp_components', + executable='component_container_mt', + output='screen', + condition=UnlessCondition(attach_to_shared_component_container_arg), + ) + + composable_node_descriptions = [ + ComposableNode( + name='resize_node', + package='isaac_ros_image_proc', + plugin='nvidia::isaac_ros::image_proc::ResizeNode', + parameters=[ + { + 'input_qos': input_qos, + 'output_width': resize_image_width, + 'output_height': resize_image_height, + 'num_blocks': num_blocks, + 'keep_aspect_ratio': keep_aspect_ratio, + 'encoding_desired': input_encoding, + } + ], + remappings=[ + ('image', image_input_topic), + ('camera_info', camera_info_input_topic), + ], + ), + ComposableNode( + name='image_format_converter_node', + package='isaac_ros_image_proc', + plugin='nvidia::isaac_ros::image_proc::ImageFormatConverterNode', + parameters=[ + { + 'image_width': network_image_width, + 'image_height': network_image_height, + 'encoding_desired': encoding_desired, + } + ], + remappings=[ + ('image_raw', 'resize/image'), + ('image', 'converted/image'), + ], + ), + ComposableNode( + name='crop_node', + package='isaac_ros_image_proc', + plugin='nvidia::isaac_ros::image_proc::CropNode', + parameters=[ + { + 'input_width': resize_image_width, + 'input_height': resize_image_height, + 'crop_width': network_image_width, + 'crop_height': network_image_height, + 'num_blocks': num_blocks, + 'crop_mode': crop_mode, + 'roi_top_left_x': int((resize_image_width - network_image_width) / 2.0), + 'roi_top_left_y': int((resize_image_height - network_image_height) / 2.0), + } + ], + remappings=[ + ('image', 'converted/image'), + ('camera_info', 'resize/camera_info'), + ], + ), + ComposableNode( + name='image_to_tensor', + package='isaac_ros_tensor_proc', + plugin='nvidia::isaac_ros::dnn_inference::ImageToTensorNode', + parameters=[ + { + 'scale': True, + 'tensor_name': 'image', + } + ], + remappings=[ + ('image', 'crop/image'), + ('tensor', 'image_tensor'), + ], + ) + ] + + if (use_planar_input): + composable_node_descriptions.extend([ + ComposableNode( + name='normalize_node', + package='isaac_ros_tensor_proc', + plugin='nvidia::isaac_ros::dnn_inference::ImageTensorNormalizeNode', + parameters=[ + { + 'mean': image_mean, + 'stddev': image_stddev, + 'input_tensor_name': 'image', + 'output_tensor_name': final_tensor_name + } + ], + remappings=[ + ('tensor', 'image_tensor'), + ], + ), + ComposableNode( + name='interleaved_to_planar_node', + package='isaac_ros_tensor_proc', + plugin='nvidia::isaac_ros::dnn_inference::InterleavedToPlanarNode', + parameters=[ + { + 'output_qos': output_qos, + 'input_tensor_shape': [network_image_height, network_image_width, 3], + 'num_blocks': num_blocks, + } + ], + remappings=[ + ('interleaved_tensor', 'normalized_tensor'), + ('planar_tensor', tensor_output_topic), + ], + ) + ] + ) + + # Skip the NHWC -> NCHW step for ShuffleSeg AMR as it's been done in the model + else: + composable_node_descriptions.append( + ComposableNode( + name='normalize_node', + package='isaac_ros_tensor_proc', + plugin='nvidia::isaac_ros::dnn_inference::ImageTensorNormalizeNode', + parameters=[ + { + 'output_qos': output_qos, + 'mean': image_mean, + 'stddev': image_stddev, + 'input_tensor_name': 'image', + 'output_tensor_name': final_tensor_name + } + ], + remappings=[ + ('tensor', 'image_tensor'), + ('normalized_tensor', tensor_output_topic), + ], + ) + ) + + load_composable_nodes = LoadComposableNodes( + target_container=component_container_name_arg, + composable_node_descriptions=composable_node_descriptions + ) + + final_launch = GroupAction( + actions=[ + dnn_image_encoder_container, + PushRosNamespace(dnn_image_encoder_namespace), + load_composable_nodes, + ], + ) + return [final_launch] + + +def generate_launch_description(): + launch_args = [ + DeclareLaunchArgument( + 'input_image_width', + default_value='0', + description='The input image width', + ), + DeclareLaunchArgument( + 'input_image_height', + default_value='0', + description='The input image height', + ), + DeclareLaunchArgument( + 'network_image_width', + default_value='0', + description='The network image width', + ), + DeclareLaunchArgument( + 'network_image_height', + default_value='0', + description='The network image height', + ), + DeclareLaunchArgument( + 'image_mean', + default_value='[0.5, 0.5, 0.5]', + description='The mean for image normalization', + ), + DeclareLaunchArgument( + 'image_stddev', + default_value='[0.5, 0.5, 0.5]', + description='The standard deviation for image normalization', + ), + DeclareLaunchArgument( + 'enable_padding', + default_value='True', + description='Whether to enable padding or not', + ), + DeclareLaunchArgument( + 'use_planar_input', + default_value='True', + description='Whether to use planar input or not', + ), + DeclareLaunchArgument( + 'input_qos', + default_value='DEFAULT', + description='The QoS settings for the input image' + ), + DeclareLaunchArgument( + 'output_qos', + default_value='DEFAULT', + description='The QoS settings for the output tensor' + ), + DeclareLaunchArgument( + 'num_blocks', + default_value='40', + description='The number of preallocated memory blocks', + ), + DeclareLaunchArgument( + 'keep_aspect_ratio', + default_value='True', + description='Whether to maintain the aspect ratio or not while resizing' + ), + DeclareLaunchArgument( + 'crop_mode', + default_value='CENTER', + description='The crop mode to crop the image using', + ), + + DeclareLaunchArgument( + 'model_name', + default_value='Vanilla', + description='The model name to use for the unet pipeline', + ), + DeclareLaunchArgument( + 'input_encoding', + default_value='rgb8', + description='The desired image format encoding', + ), + DeclareLaunchArgument( + 'encoding_desired', + default_value='rgb8', + description='The desired image format encoding', + ), + DeclareLaunchArgument( + 'final_tensor_name', + default_value='input_tensor', + description='The tensor name of the output of image encoder', + ), + DeclareLaunchArgument( + 'dnn_image_encoder_namespace', + default_value='dnn_image_encoder', + description='The namespace to put the DNN image encoder under', + ), + ] + + return LaunchDescription(launch_args + [OpaqueFunction(function=launch_setup)]) diff --git a/isaac_ros_unet/launch/isaac_ros_unet_tensor_rt.launch.py b/isaac_ros_unet/launch/isaac_ros_unet_tensor_rt.launch.py index c94c1a6..11b295e 100644 --- a/isaac_ros_unet/launch/isaac_ros_unet_tensor_rt.launch.py +++ b/isaac_ros_unet/launch/isaac_ros_unet_tensor_rt.launch.py @@ -53,6 +53,10 @@ def generate_launch_description(): 'encoder_image_stddev', default_value='[0.5, 0.5, 0.5]', description='The standard deviation for image normalization'), + DeclareLaunchArgument( + 'use_planar_input', + default_value='True', + description='Whether the input image should be in planar format or not'), DeclareLaunchArgument( 'model_file_path', default_value='', @@ -118,6 +122,7 @@ def generate_launch_description(): network_image_height = LaunchConfiguration('network_image_height') encoder_image_mean = LaunchConfiguration('encoder_image_mean') encoder_image_stddev = LaunchConfiguration('encoder_image_stddev') + use_planar_input = LaunchConfiguration('use_planar_input') # TensorRT parameters model_file_path = LaunchConfiguration('model_file_path') @@ -138,10 +143,10 @@ def generate_launch_description(): mask_height = LaunchConfiguration('mask_height') # Parameters preconfigured for PeopleSemSegNet. - encoder_dir = get_package_share_directory('isaac_ros_dnn_image_encoder') + encoder_dir = get_package_share_directory('isaac_ros_unet') encoder_node_launch = IncludeLaunchDescription( PythonLaunchDescriptionSource( - [os.path.join(encoder_dir, 'launch', 'dnn_image_encoder.launch.py')] + [os.path.join(encoder_dir, 'launch', 'isaac_ros_unet_encoder.launch.py')] ), launch_arguments={ 'input_image_width': input_image_width, @@ -151,6 +156,7 @@ def generate_launch_description(): 'image_mean': encoder_image_mean, 'image_stddev': encoder_image_stddev, 'enable_padding': 'True', + 'use_planar_input': use_planar_input, 'image_input_topic': '/image', 'camera_info_input_topic': '/camera_info', 'tensor_output_topic': '/tensor_pub', diff --git a/isaac_ros_unet/launch/isaac_ros_unet_tensor_rt_isaac_sim.launch.py b/isaac_ros_unet/launch/isaac_ros_unet_tensor_rt_isaac_sim.launch.py index acf7cbf..164770d 100644 --- a/isaac_ros_unet/launch/isaac_ros_unet_tensor_rt_isaac_sim.launch.py +++ b/isaac_ros_unet/launch/isaac_ros_unet_tensor_rt_isaac_sim.launch.py @@ -53,6 +53,10 @@ def generate_launch_description(): 'encoder_image_stddev', default_value='[0.5, 0.5, 0.5]', description='The standard deviation for image normalization'), + DeclareLaunchArgument( + 'use_planar_input', + default_value='True', + description='Whether the input image should be in planar format or not'), DeclareLaunchArgument( 'model_file_path', default_value='', @@ -114,6 +118,7 @@ def generate_launch_description(): network_image_height = LaunchConfiguration('network_image_height') encoder_image_mean = LaunchConfiguration('encoder_image_mean') encoder_image_stddev = LaunchConfiguration('encoder_image_stddev') + use_planar_input = LaunchConfiguration('use_planar_input') # Triton parameters model_file_path = LaunchConfiguration('model_file_path') @@ -134,11 +139,10 @@ def generate_launch_description(): mask_height = LaunchConfiguration('mask_height') # Parameters preconfigured for PeopleSemSegNet. - encoder_dir = get_package_share_directory('isaac_ros_dnn_image_encoder') + encoder_dir = get_package_share_directory('isaac_ros_unet') encoder_node_launch = IncludeLaunchDescription( PythonLaunchDescriptionSource( - [os.path.join(encoder_dir, 'launch', - 'dnn_image_encoder.launch.py')] + [os.path.join(encoder_dir, 'launch', 'isaac_ros_unet_encoder.launch.py')] ), launch_arguments={ 'input_image_width': input_image_width, @@ -148,6 +152,7 @@ def generate_launch_description(): 'image_mean': encoder_image_mean, 'image_stddev': encoder_image_stddev, 'enable_padding': 'True', + 'use_planar_input': use_planar_input, 'image_input_topic': '/front_stereo_camera/left/image_rect_color', 'camera_info_input_topic': '/front_stereo_camera/left/camera_info', 'tensor_output_topic': '/tensor_pub', diff --git a/isaac_ros_unet/launch/isaac_ros_unet_triton.launch.py b/isaac_ros_unet/launch/isaac_ros_unet_triton.launch.py index 23d95d9..d82f05e 100644 --- a/isaac_ros_unet/launch/isaac_ros_unet_triton.launch.py +++ b/isaac_ros_unet/launch/isaac_ros_unet_triton.launch.py @@ -53,6 +53,10 @@ def generate_launch_description(): 'encoder_image_stddev', default_value='[0.5, 0.5, 0.5]', description='The standard deviation for image normalization'), + DeclareLaunchArgument( + 'use_planar_input', + default_value='True', + description='Whether the input image should be in planar format or not'), DeclareLaunchArgument( 'model_name', default_value='', @@ -114,6 +118,7 @@ def generate_launch_description(): network_image_height = LaunchConfiguration('network_image_height') encoder_image_mean = LaunchConfiguration('encoder_image_mean') encoder_image_stddev = LaunchConfiguration('encoder_image_stddev') + use_planar_input = LaunchConfiguration('use_planar_input') # Triton parameters model_name = LaunchConfiguration('model_name') @@ -133,10 +138,10 @@ def generate_launch_description(): mask_height = LaunchConfiguration('mask_height') # Parameters preconfigured for PeopleSemSegNet. - encoder_dir = get_package_share_directory('isaac_ros_dnn_image_encoder') + encoder_dir = get_package_share_directory('isaac_ros_unet') encoder_node_launch = IncludeLaunchDescription( PythonLaunchDescriptionSource( - [os.path.join(encoder_dir, 'launch', 'dnn_image_encoder.launch.py')] + [os.path.join(encoder_dir, 'launch', 'isaac_ros_unet_encoder.launch.py')] ), launch_arguments={ 'input_image_width': input_image_width, @@ -146,6 +151,7 @@ def generate_launch_description(): 'image_mean': encoder_image_mean, 'image_stddev': encoder_image_stddev, 'enable_padding': 'True', + 'use_planar_input': use_planar_input, 'image_input_topic': '/image', 'camera_info_input_topic': '/camera_info', 'tensor_output_topic': '/tensor_pub', diff --git a/isaac_ros_unet/package.xml b/isaac_ros_unet/package.xml index 4a58b48..0f37a93 100644 --- a/isaac_ros_unet/package.xml +++ b/isaac_ros_unet/package.xml @@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0 isaac_ros_unet - 3.1.0 + 3.2.0 U-Net model processing Isaac ROS Maintainers @@ -49,6 +49,7 @@ SPDX-License-Identifier: Apache-2.0 ament_lint_common isaac_ros_test isaac_ros_tensor_rt + ament_cmake_gtest isaac_ros_tensor_rt isaac_ros_triton diff --git a/isaac_ros_unet/test/isaac_ros_unet_pol_test.py b/isaac_ros_unet/test/isaac_ros_unet_pol_test.py index 7f145be..83e10ab 100644 --- a/isaac_ros_unet/test/isaac_ros_unet_pol_test.py +++ b/isaac_ros_unet/test/isaac_ros_unet_pol_test.py @@ -74,10 +74,10 @@ def generate_test_description(): if e.errno != errno.ENOENT: print('File exists but error deleting /tmp/trt_engine.plan') - encoder_dir = get_package_share_directory('isaac_ros_dnn_image_encoder') + encoder_dir = get_package_share_directory('isaac_ros_unet') encoder_node_launch = IncludeLaunchDescription( PythonLaunchDescriptionSource( - [os.path.join(encoder_dir, 'launch', 'dnn_image_encoder.launch.py')] + [os.path.join(encoder_dir, 'launch', 'isaac_ros_unet_encoder.launch.py')] ), launch_arguments={ 'input_image_width': '1200', diff --git a/isaac_ros_unet/test/unet_decoder_node_test.cpp b/isaac_ros_unet/test/unet_decoder_node_test.cpp new file mode 100644 index 0000000..acee4f1 --- /dev/null +++ b/isaac_ros_unet/test/unet_decoder_node_test.cpp @@ -0,0 +1,126 @@ +// SPDX-FileCopyrightText: NVIDIA CORPORATION & AFFILIATES +// Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +#include +#include "unet_decoder_node.hpp" +#include "rclcpp/rclcpp.hpp" + +// Objective: to cover code lines where exceptions are thrown +// Approach: send Invalid Arguments for node parameters to trigger the exception + + +TEST(unet_decoder_node_test, test_empty_color_segmentation_mask_encoding) +{ + rclcpp::init(0, nullptr); + rclcpp::NodeOptions options; + options.append_parameter_override("color_segmentation_mask_encoding", ""); + EXPECT_THROW( + { + try { + nvidia::isaac_ros::unet::UNetDecoderNode unet_decoder_node(options); + } catch (const std::invalid_argument & e) { + EXPECT_THAT(e.what(), testing::HasSubstr("Received empty color segmentation mask encoding!")); + throw; + } catch (const rclcpp::exceptions::InvalidParameterValueException & e) { + EXPECT_THAT(e.what(), testing::HasSubstr("No parameter value set")); + throw; + } + }, std::invalid_argument); + rclcpp::shutdown(); +} + +TEST(unet_decoder_node_test, test_invalid_color_segmentation_mask_encoding) +{ + rclcpp::init(0, nullptr); + rclcpp::NodeOptions options; + options.append_parameter_override("color_segmentation_mask_encoding", "gbr8"); + options.append_parameter_override("color_palette", std::vector{1}); + EXPECT_THROW( + { + try { + nvidia::isaac_ros::unet::UNetDecoderNode unet_decoder_node(options); + } catch (const std::invalid_argument & e) { + EXPECT_THAT( + e.what(), + testing::HasSubstr("Received invalid color segmentation mask encoding")); + throw; + } catch (const rclcpp::exceptions::InvalidParameterValueException & e) { + EXPECT_THAT(e.what(), testing::HasSubstr("No parameter value set")); + throw; + } + }, std::invalid_argument); + rclcpp::shutdown(); +} + +TEST(unet_decoder_node_test, test_empty_color_palette) +{ + rclcpp::init(0, nullptr); + rclcpp::NodeOptions options; + options.append_parameter_override("color_segmentation_mask_encoding", "rgb8"); + EXPECT_THROW( + { + try { + nvidia::isaac_ros::unet::UNetDecoderNode unet_decoder_node(options); + } catch (const std::invalid_argument & e) { + EXPECT_THAT( + e.what(), + testing::HasSubstr( + "Received empty color palette! Fill this with a 24-bit hex color for each class!")); + throw; + } catch (const rclcpp::exceptions::InvalidParameterValueException & e) { + EXPECT_THAT(e.what(), testing::HasSubstr("No parameter value set")); + throw; + } + }, std::invalid_argument); + rclcpp::shutdown(); +} + +TEST(unet_decoder_node_test, test_invalid_network_output_type) +{ + rclcpp::init(0, nullptr); + rclcpp::NodeOptions options; + // options.arguments( + // { + // "--ros-args", + // "-p", "color_segmentation_mask_encoding:='rgb8'", + // "-p", "color_palette:=[1]", + // "-p", "network_output_type:='invalid'", + // }); + options.append_parameter_override("color_segmentation_mask_encoding", "rgb8"); + options.append_parameter_override("color_palette", std::vector(1)); + options.append_parameter_override("network_output_type", "invalid"); + EXPECT_THROW( + { + try { + nvidia::isaac_ros::unet::UNetDecoderNode unet_decoder_node(options); + } catch (const std::invalid_argument & e) { + EXPECT_THAT(e.what(), testing::HasSubstr("Received invalid network output type: ")); + throw; + } catch (const rclcpp::exceptions::InvalidParameterValueException & e) { + EXPECT_THAT(e.what(), testing::HasSubstr("No parameter value set")); + throw; + } + }, std::invalid_argument); + rclcpp::shutdown(); +} + + +int main(int argc, char ** argv) +{ + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/resources/peoplesemsegnet_shuffleseg_config.pbtxt b/resources/peoplesemsegnet_shuffleseg_config.pbtxt index 3ce93eb..0f3a5f5 100644 --- a/resources/peoplesemsegnet_shuffleseg_config.pbtxt +++ b/resources/peoplesemsegnet_shuffleseg_config.pbtxt @@ -11,7 +11,7 @@ input [ output [ { name: "argmax_1" - data_type: TYPE_INT32 + data_type: TYPE_INT64 dims: [ 1, 544, 960, 1 ] } ]