feat(autoware_tensorrt_bevdet): add new 3d object detection method

launch/tier4_perception_launch/launch/object_recognition/detection/detection.launch.xml

@@ -6,6 +6,7 @@
   <!-- Pipeline junctions -->
   <arg name="mode" description="options: `camera_lidar_radar_fusion`, `camera_lidar_fusion`, `lidar_radar_fusion`, `lidar` or `radar`"/>
   <arg name="lidar_detection_model" description="options: `transfusion`, `centerpoint`, `apollo`, `pointpainting`, `clustering`"/>
+  <arg name="bev_detection_model" default="bevdet" description="options: `bevdet`"/>
   <arg name="use_object_filter" description="use object filter"/>
   <arg name="objects_filter_method" description="options: `lanelet_filter` or `position_filter`"/>
   <arg name="use_pointcloud_map" description="use pointcloud map filter"/>
@@ -66,6 +67,7 @@
   <arg name="switch/detector/lidar_rule" default="false"/>
   <arg name="switch/detector/radar" default="false"/>
   <arg name="switch/detector/radar_only" default="false"/>
+  <arg name="switch/detector/camera_bev" default="false"/>
   <arg name="switch/detector/tracker_based" default="false"/>
   <let name="switch/detector/tracker_based" value="true" if="$(var use_detection_by_tracker)"/>
 
@@ -123,6 +125,9 @@
     <let name="switch/detector/radar_only" value="true" unless="$(var use_multi_channel_tracker_merger)"/>
     <let name="switch/detector/tracker_based" value="false"/>
   </group>
+  <group scoped="false" if="$(eval '&quot;$(var mode)&quot;==&quot;camera&quot;')">
+    <let name="switch/detector/camera_bev" value="true"/>
+  </group>
 
   <!-- Detector -->
   <group if="$(var switch/detector/camera_lidar)">
@@ -211,6 +216,30 @@
       <arg name="radar_object_clustering_param_path" value="$(var radar_object_clustering_param_path)"/>
     </include>
   </group>
+  <group if="$(var switch/detector/camera_bev)">
+    <!-- Camera-based bev detectors-->
+    <include file="$(find-pkg-share tier4_perception_launch)/launch/object_recognition/detection/detector/camera_bev_detector.launch.xml">
+      <arg name="input/pointcloud" value="$(var input/pointcloud)"/>
+      <arg name="input/camera0/image" value="$(var input/camera0/image)"/>
+      <arg name="input/camera0/info" value="$(var input/camera0/info)"/>
+      <arg name="input/camera1/image" value="$(var input/camera1/image)"/>
+      <arg name="input/camera1/info" value="$(var input/camera1/info)"/>
+      <arg name="input/camera2/image" value="$(var input/camera2/image)"/>
+      <arg name="input/camera2/info" value="$(var input/camera2/info)"/>
+      <arg name="input/camera3/image" value="$(var input/camera3/image)"/>
+      <arg name="input/camera3/info" value="$(var input/camera3/info)"/>
+      <arg name="input/camera4/image" value="$(var input/camera4/image)"/>
+      <arg name="input/camera4/info" value="$(var input/camera4/info)"/>
+      <arg name="input/camera5/image" value="$(var input/camera5/image)"/>
+      <arg name="input/camera5/info" value="$(var input/camera5/info)"/>
+      <arg name="input/camera6/image" value="$(var input/camera6/image)"/>
+      <arg name="input/camera6/info" value="$(var input/camera6/info)"/>
+      <arg name="input/camera7/image" value="$(var input/camera7/image)"/>
+      <arg name="input/camera7/info" value="$(var input/camera7/info)"/>
+      <arg name="number_of_cameras" value="$(var number_of_cameras)"/>
+      <arg name="bev_detection_model" value="$(var bev_detection_model)"/>
+    </include>
+  </group>
   <!-- DetectionByTracker -->
   <group if="$(var switch/detector/tracker_based)">
     <include file="$(find-pkg-share tier4_perception_launch)/launch/object_recognition/detection/detector/tracker_based_detector.launch.xml">

launch/tier4_perception_launch/launch/object_recognition/detection/detector/camera_bev_detector.launch.xml

@@ -0,0 +1,43 @@
+<?xml version="1.0"?>
+<launch>
+  <!-- Lidar parameters -->
+  <arg name="input/pointcloud"/>
+
+  <!-- Camera parameters -->
+  <arg name="input/camera0/image"/>
+  <arg name="input/camera0/info"/>
+  <arg name="input/camera1/image"/>
+  <arg name="input/camera1/info"/>
+  <arg name="input/camera2/image"/>
+  <arg name="input/camera2/info"/>
+  <arg name="input/camera3/image"/>
+  <arg name="input/camera3/info"/>
+  <arg name="input/camera4/image"/>
+  <arg name="input/camera4/info"/>
+  <arg name="input/camera5/image"/>
+  <arg name="input/camera5/info"/>
+  <arg name="input/camera6/image"/>
+  <arg name="input/camera6/info"/>
+  <arg name="input/camera7/image"/>
+  <arg name="input/camera7/info"/>
+  <arg name="number_of_cameras"/>
+  <arg name="bev_detection_model" default="bevdet" description="options: `bevdet`"/>
+
+  <!-- BEVDet -->
+  <group if="$(eval &quot;'$(var bev_detection_model)'=='bevdet'&quot;)">
+    <push-ros-namespace namespace="bevdet"/>
+    <group>
+      <include file="$(find-pkg-share tensorrt_bevdet)/launch/tensorrt_bevdet.launch.xml">
+        <arg name="input/pointcloud" value="$(var input/pointcloud)"/>
+        <arg name="input/img_fl" value="$(var input/camera0/image)"/>
+        <arg name="input/img_f" value="$(var input/camera1/image)"/>
+        <arg name="input/img_fr" value="$(var input/camera2/image)"/>
+        <arg name="input/img_bl" value="$(var input/camera3/image)"/>
+        <arg name="input/img_b" value="$(var input/camera4/image)"/>
+        <arg name="input/img_br" value="$(var input/camera5/image)"/>
+        <arg name="output/pointcloud" value="points_painting"/>
+        <arg name="output/pointcboxesloud" value="boxes"/>
+      </include>
+    </group>
+  </group>
+</launch>

launch/tier4_perception_launch/launch/perception.launch.xml

@@ -48,6 +48,7 @@
   <arg name="mode" default="camera_lidar_fusion" description="options: `camera_lidar_radar_fusion`, `camera_lidar_fusion`, `lidar_radar_fusion`, `lidar` or `radar`"/>
   <arg name="data_path" default="$(env HOME)/autoware_data" description="packages data and artifacts directory path"/>
   <arg name="lidar_detection_model" default="centerpoint" description="options: `transfusion`, `centerpoint`, `apollo`, `pointpainting`, `clustering`"/>
+  <arg name="bev_detection_model" default="bevdet" description="options: `bevdet`"/>
   <arg name="image_raw0" default="/sensing/camera/camera0/image_rect_color" description="image raw topic name"/>
   <arg name="camera_info0" default="/sensing/camera/camera0/camera_info" description="camera info topic name"/>
   <arg name="detection_rois0" default="/perception/object_recognition/detection/rois0" description="detection rois output topic name"/>
@@ -193,6 +194,7 @@
         <include file="$(find-pkg-share tier4_perception_launch)/launch/object_recognition/detection/detection.launch.xml">
           <arg name="mode" value="$(var mode)"/>
           <arg name="lidar_detection_model" value="$(var lidar_detection_model)"/>
+          <arg name="bev_detection_model" value="$(var bev_detection_model)"/>
           <arg name="use_multi_channel_tracker_merger" value="$(var use_multi_channel_tracker_merger)"/>
           <arg name="number_of_cameras" value="$(var image_number)"/>
           <arg name="input/camera0/image" value="$(var image_raw0)"/>

perception/tensorrt_bevdet/CMakeLists.txt

@@ -0,0 +1,103 @@
+cmake_minimum_required(VERSION 3.17)
+project(tensorrt_bevdet)
+
+add_compile_options(-W)
+add_compile_options(-std=c++17)
+set(CMAKE_CXX_FLAGS_RELEASE "-Wno-deprecated-declarations -O2")
+set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -std=c++17 -O0 -Xcompiler -fPIC -g -w -gencode=arch=compute_86,code=sm_86")
+find_package(tensorrt_common)
+if(NOT ${tensorrt_common_FOUND})
+  message(WARNING "The tensorrt_common package is not found. Please check its dependencies.")
+  return()
+endif()
+
+find_package(ament_cmake REQUIRED)
+find_package(cudnn_cmake_module REQUIRED)
+find_package(tensorrt_cmake_module REQUIRED)
+find_package(rclcpp REQUIRED)
+find_package(yaml-cpp REQUIRED)
+find_package(Eigen3 REQUIRED)
+find_package(OpenCV REQUIRED)
+find_package(PCL REQUIRED)
+find_package(sensor_msgs REQUIRED)
+find_package(pcl_conversions REQUIRED)
+find_package(cv_bridge REQUIRED)
+find_package(tf2_geometry_msgs REQUIRED)
+find_package(autoware_perception_msgs REQUIRED)
+
+find_package(CUDA REQUIRED)
+find_package(CUDAToolkit)
+find_package(CUDNN)
+find_package(TENSORRT)
+if(NOT (CUDAToolkit_FOUND AND CUDNN_FOUND AND TENSORRT_FOUND))
+  message(WARNING "cuda, cudnn, tensorrt libraries are not found")
+  return()
+endif()
+
+include_directories(
+  include
+  SYSTEM
+  ${YAML_CPP_INCLUDE_DIRS}
+  ${EIGEN3_INCLUDE_DIRS}
+  ${OpenCV_INCLUDE_DIRS}
+  ${PCL_INCLUDE_DIRS}
+  ${cv_bridge_INCLUDE_DIRS}
+  ${tf2_geometry_msgs_INCLUDE_DIRS}
+  ${autoware_perception_msgs_INCLUDE_DIRS}
+)
+
+cuda_add_executable(${PROJECT_NAME}_node
+    src/bevdet_node.cpp
+    src/bevdet.cpp
+    src/preprocess.cu
+    src/iou3d_nms.cu
+    src/postprocess.cu
+    src/data.cpp
+    src/cpu_jpegdecoder.cpp
+    src/nvjpegdecoder.cpp
+
+    src/preprocess_plugin.cu
+    src/bevpool_plugin.cu
+    src/alignbev_plugin.cu
+    src/gatherbev_plugin.cu
+)
+
+# Link libraries to executable
+ament_target_dependencies(${PROJECT_NAME}_node
+  "rclcpp"
+  "sensor_msgs"
+  "pcl_conversions"
+  "cv_bridge"
+  "autoware_perception_msgs"
+  "tf2_geometry_msgs"
+)
+
+target_link_libraries(${PROJECT_NAME}_node
+    yaml-cpp
+    libnvinfer.so
+    libnvonnxparser.so
+    libz.so
+    libjpeg.so
+    rclcpp::rclcpp
+    stdc++fs
+    ${NVINFER}
+    ${TENSORRT_LIBRARIES}
+    ${CUDA_LIBRARIES}
+    ${CUBLAS_LIBRARIES}
+    ${CUDNN_LIBRARY}
+    ${OpenCV_LIBS}
+    ${PCL_LIBRARIES}
+)
+
+install(TARGETS ${PROJECT_NAME}_node
+  DESTINATION lib/${PROJECT_NAME}
+)
+
+install(DIRECTORY
+  launch
+  config
+  data
+  DESTINATION share/${PROJECT_NAME}/
+)
+
+ament_package()

perception/tensorrt_bevdet/README.md

@@ -0,0 +1,40 @@
+# tensorrt_bevdet
+
+## Purpose
+
+tensorrt_bevdet is a dynamic 3D bev object detection package based on 6 surround view cameras.
+
+## Inner-workings / Algorithms
+
+BEVDet is a BEV perception algorithm based on panoramic cameras. It unifies multi-view images into the perspective of BEV for 3D object detection task. In this implementation, BEVDet network to inference with TensorRT.
+
+## Inputs / Outputs
+
+### Inputs
+
+| Name                   | Type                            | Description                                                 |
+| ---------------------- | ------------------------------- | ----------------------------------------------------------- |
+| `~/input/pointcloud`   | `sensor_msgs::msg::PointCloud2` | input pointcloud (only used for time alignment and display) |
+| `~/input/topic_img_fl` | `sensor_msgs::msg::Image`       | input front_left camera image                               |
+| `~/input/topic_img_f`  | `sensor_msgs::msg::Image`       | input front camera image                                    |
+| `~/input/topic_img_fr` | `sensor_msgs::msg::Image`       | input front_right camera image                              |
+| `~/input/topic_img_bl` | `sensor_msgs::msg::Image`       | input back_left camera image                                |
+| `~/input/topic_img_b`  | `sensor_msgs::msg::Image`       | input back camera image                                     |
+| `~/input/topic_img_br` | `sensor_msgs::msg::Image`       | input back_right camera image                               |
+
+### Outputs
+
+| Name                  | Type                                             | Description                               |
+| --------------------- | ------------------------------------------------ | ----------------------------------------- |
+| `~/output/boxes`      | `autoware_perception_msgs::msg::DetectedObjects` | detected objects                          |
+| `~/output/pointcloud` | `sensor_msgs::msg::PointCloud2`                  | output pointcloud (only used for display) |
+
+## Limittation
+
+The model is trained on open-source dataset `NuScenes` and has poor generalization on its own dataset, If you want to use this model to infer your data, you need to retrain it.
+
+You can traine models by refer below links.
+
+[BEVDet](https://github.com/HuangJunJie2017/BEVDet/tree/dev2.1)
+
+[BEVDet export onnx](https://github.com/LCH1238/BEVDet/tree/export)

perception/tensorrt_bevdet/config/bevdet.param.yaml

@@ -0,0 +1,11 @@
+/**:
+    ros__parameters:
+      data_params:
+        N: 6  # camera num
+        H: 900 # image height
+        W: 1600 # image width
+        cams: ["CAM_FRONT_LEFT", "CAM_FRONT", "CAM_FRONT_RIGHT", "CAM_BACK_LEFT", "CAM_BACK", "CAM_BACK_RIGHT"]
+      post_process_params:
+        # post-process params
+        score_thre: 0.2
+        class_names: ["car", "truck", "construction_vehicle", "bus", "trailer", "barrier", "motorcycle", "bicycle", "pedestrian", "traffic_cone"]

perception/tensorrt_bevdet/config/bevdet_r50_4dlongterm_depth.yaml

@@ -0,0 +1,62 @@
+bev_range: [-51.2, -51.2, -5.0, 51.2, 51.2, 3.0]
+mean: [123.675, 116.28, 103.53]
+std: [58.395, 57.12, 57.375]
+use_depth: true
+use_adj: true
+adj_num: 8
+transform_size: 6
+cam_params_size: 27
+sampling: nearest # nearest # or bicubic
+data_config:
+  Ncams: 6
+  cams: [CAM_FRONT_LEFT, CAM_FRONT, CAM_FRONT_RIGHT, CAM_BACK_LEFT, CAM_BACK, CAM_BACK_RIGHT]
+  crop: [140, 0]
+  flip: true
+  input_size: [256, 704]
+  resize: [-0.06, 0.11]
+  resize_test: 0.0
+  rot: [-5.4, 5.4]
+  src_size: [900, 1600]
+grid_config:
+  depth: [1.0, 60.0, 0.5]
+  x: [-51.2, 51.2, 0.8]
+  y: [-51.2, 51.2, 0.8]
+  z: [-5, 3, 8]
+model:
+  bevpool_channels: 80
+  coder:
+    code_size: 9
+    max_num: 500
+    post_center_range: [-61.2, -61.2, -10.0, 61.2, 61.2, 10.0]
+    score_threshold: 0.1
+  common_head:
+    channels: [2, 1, 3, 2, 2]
+    names: [reg, height, dim, rot, vel]
+  down_sample: 16
+  tasks:
+    - class_names:
+        [
+          car,
+          truck,
+          construction_vehicle,
+          bus,
+          trailer,
+          barrier,
+          motorcycle,
+          bicycle,
+          pedestrian,
+          traffic_cone,
+        ]
+      num_class: 10
+test_cfg:
+  max_per_img: 500
+  max_pool_nms: false
+  min_radius: [4, 12, 10, 1, 0.85, 0.175]
+  nms_rescale_factor:
+    - [1.0, 0.7, 0.7, 0.4, 0.55, 1.1, 1.0, 1.0, 1.5, 3.5]
+  nms_thr: [0.2]
+  nms_type: [rotate]
+  post_center_limit_range: [-61.2, -61.2, -10.0, 61.2, 61.2, 10.0]
+  post_max_size: 500
+  pre_max_size: 1000
+  score_threshold: 0.1