Add object detection capability and python API

AirLib/include/api/RpcLibAdaptorsBase.hpp

@@ -40,6 +40,26 @@ namespace airlib_rpclib
                 d.push_back(TDest(s.at(i)));
         }
 
+        struct Vector2r
+        {
+            msr::airlib::real_T x_val = 0, y_val = 0;
+            MSGPACK_DEFINE_MAP(x_val, y_val);
+
+            Vector2r()
+            {
+            }
+
+            Vector2r(const msr::airlib::Vector2r& s)
+            {
+                x_val = s.x();
+                y_val = s.y();
+            }
+            msr::airlib::Vector2r to() const
+            {
+                return msr::airlib::Vector2r(x_val, y_val);
+            }
+        };
+
         struct Vector3r
         {
             msr::airlib::real_T x_val = 0, y_val = 0, z_val = 0;
@@ -236,6 +256,115 @@ namespace airlib_rpclib
             }
         };
 
+        struct Box2D
+        {
+            Vector2r min;
+            Vector2r max;
+
+            MSGPACK_DEFINE_MAP(min, max);
+
+            Box2D()
+            {
+            }
+
+            Box2D(const msr::airlib::Box2D& s)
+            {
+                min = s.min;
+                max = s.max;
+            }
+
+            msr::airlib::Box2D to() const
+            {
+                msr::airlib::Box2D s;
+                s.min = min.to();
+                s.max = max.to();
+
+                return s;
+            }
+        };
+
+        struct Box3D
+        {
+            Vector3r min;
+            Vector3r max;
+
+            MSGPACK_DEFINE_MAP(min, max);
+
+            Box3D()
+            {
+            }
+
+            Box3D(const msr::airlib::Box3D& s)
+            {
+                min = s.min;
+                max = s.max;
+            }
+
+            msr::airlib::Box3D to() const
+            {
+                msr::airlib::Box3D s;
+                s.min = min.to();
+                s.max = max.to();
+
+                return s;
+            }
+        };
+
+        struct DetectionInfo
+        {
+            std::string name;
+            GeoPoint geo_point;
+            Box2D box2D;
+            Box3D box3D;
+            Pose relative_pose;
+
+            MSGPACK_DEFINE_MAP(name, geo_point, box2D, box3D, relative_pose);
+
+            DetectionInfo()
+            {
+            }
+
+            DetectionInfo(const msr::airlib::DetectionInfo& d)
+            {
+                name = d.name;
+                geo_point = d.geo_point;
+                box2D = d.box2D;
+                box3D = d.box3D;
+                relative_pose = d.relative_pose;
+            }
+
+            msr::airlib::DetectionInfo to() const
+            {
+                msr::airlib::DetectionInfo d;
+                d.name = name;
+                d.geo_point = geo_point.to();
+                d.box2D = box2D.to();
+                d.box3D = box3D.to();
+                d.relative_pose = relative_pose.to();
+
+                return d;
+            }
+
+            static std::vector<DetectionInfo> from(
+                const std::vector<msr::airlib::DetectionInfo>& request)
+            {
+                std::vector<DetectionInfo> request_adaptor;
+                for (const auto& item : request)
+                    request_adaptor.push_back(DetectionInfo(item));
+
+                return request_adaptor;
+            }
+            static std::vector<msr::airlib::DetectionInfo> to(
+                const std::vector<DetectionInfo>& request_adapter)
+            {
+                std::vector<msr::airlib::DetectionInfo> request;
+                for (const auto& item : request_adapter)
+                    request.push_back(item.to());
+
+                return request;
+            }
+        };
+
         struct CameraInfo
         {
             Pose pose;

AirLib/include/api/RpcLibClientBase.hpp

@@ -74,6 +74,11 @@ namespace airlib
         int simGetSegmentationObjectID(const std::string& mesh_name) const;
         void simPrintLogMessage(const std::string& message, std::string message_param = "", unsigned char severity = 0);
 
+        void simAddDetectionFilterMeshName(const std::string& camera_name, const std::string& mesh_name, const std::string& vehicle_name = "");
+        void simSetDetectionFilterRadius(const std::string& camera_name, const float radius_cm, const std::string& vehicle_name = "");
+        void simClearDetectionMeshNames(const std::string& camera_name, const std::string& vehicle_name = "");
+        vector<DetectionInfo> simGetDetections(const std::string& camera_name, ImageCaptureBase::ImageType image_type, const std::string& vehicle_name = "");
+
         void simFlushPersistentMarkers();
         void simPlotPoints(const vector<Vector3r>& points, const vector<float>& color_rgba, float size, float duration, bool is_persistent);
         void simPlotLineStrip(const vector<Vector3r>& points, const vector<float>& color_rgba, float thickness, float duration, bool is_persistent);

AirLib/include/api/VehicleSimApiBase.hpp

@@ -71,6 +71,11 @@ namespace airlib
         virtual void toggleTrace() = 0;
         virtual void setTraceLine(const std::vector<float>& color_rgba, float thickness) = 0;
 
+        virtual void addDetectionFilterMeshName(const std::string& camera_name, ImageCaptureBase::ImageType image_type, const std::string& mesh_name) = 0;
+        virtual void setDetectionFilterRadius(const std::string& camera_name, ImageCaptureBase::ImageType image_type, const float radius_cm) = 0;
+        virtual void clearDetectionMeshNames(const std::string& camera_name, ImageCaptureBase::ImageType image_type) = 0;
+        virtual std::vector<DetectionInfo> getDetections(const std::string& camera_name, ImageCaptureBase::ImageType image_type) const = 0;
+
         //use pointer here because of derived classes for VehicleSetting
         const AirSimSettings::VehicleSetting* getVehicleSetting() const
         {

AirLib/include/common/CommonStructs.hpp

@@ -270,6 +270,54 @@ namespace airlib
         }
     };
 
+    struct Box2D
+    {
+        Vector2r min;
+        Vector2r max;
+
+        Box2D()
+        {
+        }
+
+        Box2D(Vector2r min_val, Vector2r max_val)
+            : min(min_val), max(max_val)
+        {
+        }
+    };
+
+    struct Box3D
+    {
+        Vector3r min;
+        Vector3r max;
+
+        Box3D()
+        {
+        }
+
+        Box3D(Vector3r min_val, Vector3r max_val)
+            : min(min_val), max(max_val)
+        {
+        }
+    };
+
+    struct DetectionInfo
+    {
+        std::string name = "";
+        GeoPoint geo_point = GeoPoint();
+        Box2D box2D = Box2D();
+        Box3D box3D = Box3D();
+        Pose relative_pose = Pose();
+
+        DetectionInfo()
+        {
+        }
+
+        DetectionInfo(const std::string& name_val, const GeoPoint& geo_point_val, const Box2D& box2D_val, const Box3D& box3D_val, const Pose& relative_pose_val)
+            : name(name_val), geo_point(geo_point_val), box2D(box2D_val), box3D(box3D_val), relative_pose(relative_pose_val)
+        {
+        }
+    };
+
     struct CollisionResponse
     {
         unsigned int collision_count_raw = 0;

AirLib/include/vehicles/car/api/CarRpcLibAdaptors.hpp

@@ -87,6 +87,7 @@ namespace airlib_rpclib
             }
         };
     };
+
 }
 } //namespace
 

AirLib/include/vehicles/multirotor/api/MultirotorRpcLibAdaptors.hpp

@@ -138,6 +138,7 @@ namespace airlib_rpclib
             }
         };
     };
+
 }
 } //namespace
 

AirLib/src/api/RpcLibClientBase.cpp

@@ -213,6 +213,24 @@ __pragma(warning(disable : 4239))
             return pimpl_->client.call("simGetSegmentationObjectID", mesh_name).as<int>();
         }
 
+        void RpcLibClientBase::simAddDetectionFilterMeshName(const std::string& camera_name, const std::string& mesh_name, const std::string& vehicle_name)
+        {
+            pimpl_->client.call("simAddDetectionFilterMeshName", camera_name, mesh_name, vehicle_name);
+        }
+        void RpcLibClientBase::simSetDetectionFilterRadius(const std::string& camera_name, const float radius_cm, const std::string& vehicle_name)
+        {
+            pimpl_->client.call("simSetDetectionFilterRadius", camera_name, radius_cm, vehicle_name);
+        }
+        void RpcLibClientBase::simClearDetectionMeshNames(const std::string& camera_name, const std::string& vehicle_name)
+        {
+            pimpl_->client.call("simClearDetectionMeshNames", camera_name, vehicle_name);
+        }
+        vector<DetectionInfo> RpcLibClientBase::simGetDetections(const std::string& camera_name, ImageCaptureBase::ImageType image_type, const std::string& vehicle_name)
+        {
+            const auto& result = pimpl_->client.call("simGetDetections", camera_name, image_type, vehicle_name).as<vector<RpcLibAdaptorsBase::DetectionInfo>>();
+            return RpcLibAdaptorsBase::DetectionInfo::to(result);
+        }
+
         CollisionInfo RpcLibClientBase::simGetCollisionInfo(const std::string& vehicle_name) const
         {
             return pimpl_->client.call("simGetCollisionInfo", vehicle_name).as<RpcLibAdaptorsBase::CollisionInfo>().to();

AirLib/src/api/RpcLibServerBase.cpp

@@ -182,6 +182,20 @@ namespace airlib
             return getWorldSimApi()->getSegmentationObjectID(mesh_name);
         });
 
+        pimpl_->server.bind("simAddDetectionFilterMeshName", [&](const std::string& camera_name, ImageCaptureBase::ImageType type, const std::string& mesh_name, const std::string& vehicle_name) -> void {
+            getVehicleSimApi(vehicle_name)->addDetectionFilterMeshName(camera_name, type, mesh_name);
+        });
+        pimpl_->server.bind("simSetDetectionFilterRadius", [&](const std::string& camera_name, ImageCaptureBase::ImageType type, const float radius_cm, const std::string& vehicle_name) -> void {
+            getVehicleSimApi(vehicle_name)->setDetectionFilterRadius(camera_name, type, radius_cm);
+        });
+        pimpl_->server.bind("simClearDetectionMeshNames", [&](const std::string& camera_name, ImageCaptureBase::ImageType type, const std::string& vehicle_name) -> void {
+            getVehicleSimApi(vehicle_name)->clearDetectionMeshNames(camera_name, type);
+        });
+        pimpl_->server.bind("simGetDetections", [&](const std::string& camera_name, ImageCaptureBase::ImageType type, const std::string& vehicle_name) -> vector<RpcLibAdaptorsBase::DetectionInfo> {
+            const auto& response = getVehicleSimApi(vehicle_name)->getDetections(camera_name, type);
+            return RpcLibAdaptorsBase::DetectionInfo::from(response);
+        });
+
         pimpl_->server.bind("reset", [&]() -> void {
             //Exit if already resetting.
             static bool resetInProgress;