Merge branch 'develop' of https://github.com/PaddlePaddle/FastDeploy into develop

Author: yunyaoXYY

.gitignore

@@ -41,4 +41,5 @@ yalc.*
 examples/vision/collect_quantize_cc.sh
 examples/vision/tests_quantize
 fastdeploy/LICENSE
-fastdeploy/ThirdPartyNotices.txt
+fastdeploy/ThirdPartyNotices.txt
+FastDeployCSharp.cmake

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "poros/third_party/googletest"]
+	path = poros/third_party/googletest
+	url = https://github.com/google/googletest.git
+[submodule "poros/third_party/gflags"]
+	path = poros/third_party/gflags
+	url = https://github.com/gflags/gflags.git

CMakeLists.txt

@@ -79,12 +79,12 @@ option(WITH_CSHARPAPI "Whether to compile with c# api" OFF)
 ############################# Options for Android cross compiling #########################
 if(ANDROID)
   option(WITH_OPENCV_STATIC "Whether to use OpenCV static lib for Android." OFF)
-  option(WITH_LITE_STATIC "Whether to use OpenCV Paddle Lite static lib for Android." OFF)
-  option(WITH_FLYCV_STATIC "Whether to use OpenCV FlyCV static lib for Android." OFF)
-  option(WITH_OPENMP "Whether to use OpenCV OpenMP support for Android." OFF)
-  option(WITH_JAVA "Whether to build jni lib for Android." OFF)
+  option(WITH_LITE_STATIC "Whether to use Paddle Lite static lib for Android." OFF)
+  option(WITH_OPENMP "Whether to use OpenMP support for Android." OFF)
+  option(WITH_JAVA "Whether to build JNI lib for Android." OFF)
   option(WITH_STATIC_LIB "Whether to build FastDeploy static lib." OFF)
 endif()
+
 # Whether to build fastdeploy with vision/text/... examples, only for testings.
 option(BUILD_EXAMPLES "Whether to build fastdeploy with vision examples" OFF)
 

FastDeploy.cmake.in

@@ -48,7 +48,6 @@ include(${CMAKE_CURRENT_LIST_DIR}/utils.cmake)
 
 # Set C++11 as standard for the whole project
 if(NOT MSVC)
-  set(CMAKE_CXX_STANDARD 11)
   set(CMAKE_CXX_FLAGS "-Wno-format")
   add_definitions(-D_GLIBCXX_USE_CXX11_ABI=1)
 endif(NOT MSVC)
@@ -131,9 +130,9 @@ endif()
 
 if(ENABLE_RKNPU2_BACKEND)
   if(RKNN2_TARGET_SOC STREQUAL "RK356X")
-    set(RKNPU2_LIB ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/rknpu2_runtime/RK356X/lib/librknn_api.so)
+    set(RKNPU2_LIB ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/rknpu2_runtime/lib/librknnrt.so)
   elseif (RKNN2_TARGET_SOC STREQUAL "RK3588")
-    set(RKNPU2_LIB ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/rknpu2_runtime/RK3588/lib/librknn_api.so)
+    set(RKNPU2_LIB ${CMAKE_CURRENT_LIST_DIR}/third_libs/install/rknpu2_runtime/lib/librknnrt.so)
   else ()
     message(FATAL_ERROR "RKNN2_TARGET_SOC is not set, ref value: RK356X or RK3588")
   endif()

README_CN.md

@@ -44,7 +44,7 @@
 </div>
 
 
- **⚡️FastDeploy**是一款**全场景**、**易用灵活**、**极致高效**的AI推理部署工具， 支持**云边端**部署。提供超过 🔥160+ **Text**，**Vision**， **Speech**和**跨模态**模型📦**开箱即用**的部署体验，并实现🔚**端到端**的推理性能优化。包括 [物体检测](./examples/vision/detection)、[字符识别（OCR）](./examples/vision/ocr)、[人脸](./examples/vision/facedet)、[人像扣图](./examples/vision/matting)、[多目标跟踪系统](./examples/vision/tracking/pptracking)、[NLP](./examples/text)、[Stable Difussion文图生成](./examples/multimodal/stable_diffusion)、[TTS](./examples/audio/pp-tts) 等几十种任务场景，满足开发者**多场景、多硬件、多平台**的产业部署需求。
+ **⚡️FastDeploy**是一款**全场景**、**易用灵活**、**极致高效**的AI推理部署工具， 支持**云边端**部署。提供超过 🔥160+ **Text**，**Vision**， **Speech**和**跨模态**模型📦**开箱即用**的部署体验，并实现🔚**端到端**的推理性能优化。包括 [物体检测](./examples/vision/detection)、[字符识别（OCR）](./examples/vision/ocr)、[人脸](./examples/vision/facedet)、[人像扣图](./examples/vision/matting)、[多目标跟踪系统](./examples/vision/tracking/pptracking)、[NLP](./examples/text)、[Stable Diffusion文图生成](./examples/multimodal/stable_diffusion)、[TTS](./examples/audio/pp-tts) 等几十种任务场景，满足开发者**多场景、多硬件、多平台**的产业部署需求。
 
 <div align="center">
 

README_EN.md

@@ -41,7 +41,7 @@ English | [简体中文](README_CN.md) | [हिन्दी](./docs/docs_i18n/R
 </div>
 
 **⚡️FastDeploy** is an **Easy-to-use** and **High Performance** AI model deployment toolkit for Cloud, Mobile and Edge with 📦**out-of-the-box and unified experience**, 🔚**end-to-end optimization** for over **🔥160+ Text, Vision, Speech and Cross-modal AI models**.
-Including [image classification](examples/vision/classification), [object detection](examples/vision/detection), [OCR](./examples/vision/ocr), [face detection](./examples/vision/facedet), [matting](./examples/vision/matting), [pp-tracking](./examples/vision/tracking/pptracking), [NLP](./examples/text), [stable difussion](./examples/multimodal/stable_diffusion), [TTS](./examples/audio/pp-tts) and other tasks to meet developers' industrial deployment needs for **multi-scenario**, **multi-hardware** and **multi-platform**.
+Including [image classification](examples/vision/classification), [object detection](examples/vision/detection), [OCR](./examples/vision/ocr), [face detection](./examples/vision/facedet), [matting](./examples/vision/matting), [pp-tracking](./examples/vision/tracking/pptracking), [NLP](./examples/text), [stable diffusion](./examples/multimodal/stable_diffusion), [TTS](./examples/audio/pp-tts) and other tasks to meet developers' industrial deployment needs for **multi-scenario**, **multi-hardware** and **multi-platform**.
 
 <div align="center">
 

benchmark/cpp/CMakeLists.txt

@@ -12,15 +12,31 @@ add_executable(benchmark_yolov5 ${PROJECT_SOURCE_DIR}/benchmark_yolov5.cc)
 add_executable(benchmark_ppyolov8 ${PROJECT_SOURCE_DIR}/benchmark_ppyolov8.cc)
 add_executable(benchmark_ppcls ${PROJECT_SOURCE_DIR}/benchmark_ppcls.cc)
 add_executable(benchmark_precision_ppyolov8 ${PROJECT_SOURCE_DIR}/benchmark_precision_ppyolov8.cc)
+add_executable(benchmark_ppseg ${PROJECT_SOURCE_DIR}/benchmark_ppseg.cc)
+add_executable(benchmark_ppocr_det ${PROJECT_SOURCE_DIR}/benchmark_ppocr_det.cc)
+add_executable(benchmark_ppocr_cls ${PROJECT_SOURCE_DIR}/benchmark_ppocr_cls.cc)
+add_executable(benchmark_ppocr_rec ${PROJECT_SOURCE_DIR}/benchmark_ppocr_rec.cc)
 
 if(UNIX AND (NOT APPLE) AND (NOT ANDROID))
   target_link_libraries(benchmark_yolov5 ${FASTDEPLOY_LIBS} gflags pthread)
   target_link_libraries(benchmark_ppyolov8 ${FASTDEPLOY_LIBS} gflags pthread)
   target_link_libraries(benchmark_ppcls ${FASTDEPLOY_LIBS} gflags pthread)
   target_link_libraries(benchmark_precision_ppyolov8 ${FASTDEPLOY_LIBS} gflags pthread)
+  target_link_libraries(benchmark_ppseg ${FASTDEPLOY_LIBS} gflags pthread)
+  target_link_libraries(benchmark_ppocr_det ${FASTDEPLOY_LIBS} gflags pthread)
+  target_link_libraries(benchmark_ppocr_cls ${FASTDEPLOY_LIBS} gflags pthread)
+  target_link_libraries(benchmark_ppocr_rec ${FASTDEPLOY_LIBS} gflags pthread)
 else()
   target_link_libraries(benchmark_yolov5 ${FASTDEPLOY_LIBS} gflags)
   target_link_libraries(benchmark_ppyolov8 ${FASTDEPLOY_LIBS} gflags)
   target_link_libraries(benchmark_ppcls ${FASTDEPLOY_LIBS} gflags)
   target_link_libraries(benchmark_precision_ppyolov8 ${FASTDEPLOY_LIBS} gflags)
+  target_link_libraries(benchmark_ppseg ${FASTDEPLOY_LIBS} gflags)
+  target_link_libraries(benchmark_ppocr_det ${FASTDEPLOY_LIBS} gflags)
+  target_link_libraries(benchmark_ppocr_cls ${FASTDEPLOY_LIBS} gflags)
+  target_link_libraries(benchmark_ppocr_rec ${FASTDEPLOY_LIBS} gflags)
+endif()
+# only for Android ADB test
+if(ANDROID)
+  install_fastdeploy_libraries(${CMAKE_CURRENT_BINARY_DIR})
 endif()

benchmark/cpp/README.md

@@ -0,0 +1,137 @@
+# FastDeploy C++ Benchmarks
+
+## 1. 编译选项  
+以下选项为benchmark相关的编译选项，在编译用来跑benchmark的sdk时，必须开启。  
+
+|选项|需要设置的值|说明|
+|---|---|---|  
+| ENABLE_BENCHMARK  | ON | 默认OFF, 是否打开BENCHMARK模式 |
+| ENABLE_VISION     | ON | 默认OFF，是否编译集成视觉模型的部署模块 |
+| ENABLE_TEXT       | ON | 默认OFF，是否编译集成文本NLP模型的部署模块 |  
+
+运行FastDeploy C++ Benchmark，需先准备好相应的环境，并在ENABLE_BENCHMARK=ON模式下从源码编译FastDeploy C++ SDK. 以下将按照硬件维度，来说明相应的系统环境要求。不同环境下的详细要求，请参考[FastDeploy环境要求](../../docs/cn/build_and_install)  
+
+## 2. Benchmark 参数设置说明  
+
+<div id="参数设置说明"></div>  
+
+
+| 参数                 | 作用                                        |
+| -------------------- | ------------------------------------------ |
+| --model              | 模型路径                                     |
+| --image              | 图片路径    |
+| --device             | 选择 CPU/GPU/XPU，默认为 CPU  |
+| --cpu_thread_nums     | CPU 线程数，默认为 8      |
+| --device_id          | GPU/XPU 卡号，默认为 0 |
+| --warmup           | 跑benchmark的warmup次数，默认为 200 |
+| --repeat           | 跑benchmark的循环次数，默认为 1000 |  
+| --profile_mode      | 指定需要测试性能的模式，可选值为`[runtime, end2end]`，默认为 runtime |  
+| --include_h2d_d2h   | 是否把H2D+D2H的耗时统计在内，该参数只在profile_mode为runtime时有效，默认为 false |  
+| --backend            | 指定后端类型，有default, ort, ov, trt, paddle, paddle_trt, lite 等，为default时，会自动选择最优后端，推荐设置为显式设置明确的backend。默认为 default   |
+| --use_fp16    | 是否开启fp16，当前只对 trt, paddle-trt, lite后端有效，默认为 false |
+| --collect_memory_info    | 是否记录 cpu/gpu memory信息，默认 false  |
+| --sampling_interval    | 记录 cpu/gpu memory信息采样时间间隔，单位ms，默认为 50  |  
+
+## 3. X86_64 CPU 和 NVIDIA GPU 环境下运行 Benchmark
+
+### 3.1 环境准备  
+
+Linux上编译需满足:
+  - gcc/g++ >= 5.4(推荐8.2)
+  - cmake >= 3.18.0
+  - CUDA >= 11.2
+  - cuDNN >= 8.2
+  - TensorRT >= 8.5
+
+在GPU上编译FastDeploy需要准备好相应的CUDA环境以及TensorRT，详细文档请参考[GPU编译文档](https://github.com/PaddlePaddle/FastDeploy/blob/develop/docs/cn/build_and_install/gpu.md)。  
+
+### 3.2 编译FastDeploy C++ SDK  
+```bash
+# 源码编译SDK
+git clone https://github.com/PaddlePaddle/FastDeploy.git -b develop
+cd FastDeploy
+mkdir build && cd build
+cmake .. -DWITH_GPU=ON \
+         -DENABLE_ORT_BACKEND=ON \
+         -DENABLE_PADDLE_BACKEND=ON \
+         -DENABLE_OPENVINO_BACKEND=ON \
+         -DENABLE_TRT_BACKEND=ON \
+         -DENABLE_VISION=ON \
+         -DENABLE_TEXT=ON \
+         -DENABLE_BENCHMARK=ON \  # 开启benchmark模式
+         -DTRT_DIRECTORY=/Paddle/TensorRT-8.5.2.2 \
+         -DCUDA_DIRECTORY=/usr/local/cuda \
+         -DCMAKE_INSTALL_PREFIX=${PWD}/compiled_fastdeploy_sdk
+
+make -j12
+make install  
+
+# 配置SDK路径
+cd ..  
+export FD_GPU_SDK=${PWD}/build/compiled_fastdeploy_sdk
+```  
+### 3.3 编译 Benchmark 示例  
+```bash  
+cd benchmark/cpp
+mkdir build && cd build  
+cmake .. -DFASTDEPLOY_INSTALL_DIR=${FD_GPU_SDK}  
+make -j4
+```
+
+### 3.4 运行 Benchmark 示例  
+
+在X86 CPU + NVIDIA GPU下，FastDeploy 目前支持多种推理后端，下面以 PaddleYOLOv8 为例，跑出多后端在 CPU/GPU 对应 benchmark 数据。
+
+- 下载模型文件和测试图片  
+```bash  
+wget https://bj.bcebos.com/paddlehub/fastdeploy/yolov8_s_500e_coco.tgz  
+wget https://gitee.com/paddlepaddle/PaddleDetection/raw/release/2.4/demo/000000014439.jpg
+tar -zxvf yolov8_s_500e_coco.tgz
+```
+
+- 运行 yolov8 benchmark 示例  
+
+```bash  
+
+# 统计性能  
+# CPU
+# Paddle Inference
+./benchmark_ppyolov8 --model yolov8_s_500e_coco --image 000000014439.jpg --device cpu --cpu_thread_nums 8 --backend paddle --profile_mode runtime
+
+# ONNX Runtime
+./benchmark_ppyolov8 --model yolov8_s_500e_coco --image 000000014439.jpg --device cpu --cpu_thread_nums 8 --backend ort --profile_mode runtime
+
+# OpenVINO
+./benchmark_ppyolov8 --model yolov8_s_500e_coco --image 000000014439.jpg --device cpu --cpu_thread_nums 8 --backend ov --profile_mode runtime
+
+# GPU
+# Paddle Inference
+./benchmark_ppyolov8 --model yolov8_s_500e_coco --image 000000014439.jpg --device gpu --device_id 0 --backend paddle --profile_mode runtime --warmup 200 --repeat 2000
+
+# Paddle Inference + TensorRT
+./benchmark_ppyolov8 --model yolov8_s_500e_coco --image 000000014439.jpg --device gpu --device_id 0 --backend paddle_trt --profile_mode runtime --warmup 200 --repeat 2000
+
+# Paddle Inference + TensorRT + FP16
+./benchmark_ppyolov8 --model yolov8_s_500e_coco --image 000000014439.jpg --device gpu --device_id 0 --backend paddle --profile_mode runtime --warmup 200 --repeat 2000 --use_fp16
+
+# ONNX Runtime
+./benchmark_ppyolov8 --model yolov8_s_500e_coco --image 000000014439.jpg --device gpu --device_id 0 --backend ort --profile_mode runtime --warmup 200 --repeat 2000
+
+# TensorRT
+./benchmark_ppyolov8 --model yolov8_s_500e_coco --image 000000014439.jpg --device gpu --device_id 0 --backend paddle --profile_mode runtime --warmup 200 --repeat 2000
+
+# TensorRT + FP16
+./benchmark_ppyolov8 --model yolov8_s_500e_coco --image 000000014439.jpg --device gpu --device_id 0 --backend trt --profile_mode runtime --warmup 200 --repeat 2000 --use_fp16
+
+# 统计内存显存占用  
+# 增加--collect_memory_info选项
+./benchmark_ppyolov8 --model yolov8_s_500e_coco --image 000000014439.jpg --device cpu --cpu_thread_nums 8 --backend paddle --profile_mode runtime --collect_memory_info
+```
+注意，为避免对性能统计产生影响，测试性能时，最好不要开启内存显存统计的功能，当指定--collect_memory_info参数时，只有内存显存参数是稳定可靠的。更多参数设置，请参考[参数设置说明](#参数设置说明)
+
+
+## 4. ARM CPU 环境下运行 Benchmark
+- TODO
+
+## 5. 昆仑芯 XPU 环境下运行 Benchmark
+- TODO

benchmark/cpp/benchmark_ppcls.cc

@@ -16,21 +16,47 @@
 #include "macros.h"
 #include "option.h"
 
+namespace vision = fastdeploy::vision;
+namespace benchmark = fastdeploy::benchmark;
+
 int main(int argc, char* argv[]) {
+#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
   // Initialization
   auto option = fastdeploy::RuntimeOption();
   if (!CreateRuntimeOption(&option, argc, argv, true)) {
     return -1;
   }
   auto im = cv::imread(FLAGS_image);
   // Set max_batch_size 1 for best performance
-  option.trt_option.max_batch_size = 1;
+  if (FLAGS_backend == "paddle_trt") {
+    option.trt_option.max_batch_size = 1;
+  }
   auto model_file = FLAGS_model + sep + "inference.pdmodel";
   auto params_file = FLAGS_model + sep + "inference.pdiparams";
   auto config_file = FLAGS_model + sep + "inference_cls.yaml";
-  auto model_ppcls = fastdeploy::vision::classification::PaddleClasModel(
+  auto model_ppcls = vision::classification::PaddleClasModel(
       model_file, params_file, config_file, option);
-  fastdeploy::vision::ClassifyResult res;
+  vision::ClassifyResult res;
+  // Run once at least
+  model_ppcls.Predict(im, &res);
+  // 1. Test result diff
+  std::cout << "=============== Test result diff =================\n";
+  // Save result to -> disk.
+  std::string cls_result_path = "ppcls_result.txt";
+  benchmark::ResultManager::SaveClassifyResult(res, cls_result_path);
+  // Load result from <- disk.
+  vision::ClassifyResult res_loaded;
+  benchmark::ResultManager::LoadClassifyResult(&res_loaded, cls_result_path);
+  // Calculate diff between two results.
+  auto cls_diff =
+      benchmark::ResultManager::CalculateDiffStatis(res, res_loaded);
+  std::cout << "Labels diff: mean=" << cls_diff.labels.mean
+            << ", max=" << cls_diff.labels.max
+            << ", min=" << cls_diff.labels.min << std::endl;
+  std::cout << "Scores diff: mean=" << cls_diff.scores.mean
+            << ", max=" << cls_diff.scores.max
+            << ", min=" << cls_diff.scores.min << std::endl;
   BENCHMARK_MODEL(model_ppcls, model_ppcls.Predict(im, &res))
+#endif
   return 0;
 }

benchmark/cpp/benchmark_ppocr_cls.cc

@@ -0,0 +1,57 @@
+// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "flags.h"
+#include "macros.h"
+#include "option.h"
+
+int main(int argc, char* argv[]) {
+#if defined(ENABLE_BENCHMARK) && defined(ENABLE_VISION)
+  // Initialization
+  auto option = fastdeploy::RuntimeOption();
+  if (!CreateRuntimeOption(&option, argc, argv, true)) {
+    return -1;
+  }
+  auto im = cv::imread(FLAGS_image);
+  // Classification Model
+  auto cls_model_file = FLAGS_model + sep + "inference.pdmodel";
+  auto cls_params_file = FLAGS_model + sep + "inference.pdiparams";
+  if (FLAGS_backend == "paddle_trt") {
+    option.paddle_infer_option.collect_trt_shape = true;
+  }
+  if (FLAGS_backend == "paddle_trt" || FLAGS_backend == "trt") {
+    option.trt_option.SetShape("x", {1, 3, 48, 10}, {4, 3, 48, 320},
+                               {8, 3, 48, 1024});
+  }
+  auto model_ppocr_cls = fastdeploy::vision::ocr::Classifier(
+      cls_model_file, cls_params_file, option);
+  int32_t res_label;
+  float res_score;
+  // Run once at least
+  model_ppocr_cls.Predict(im, &res_label, &res_score);
+  // 1. Test result diff
+  std::cout << "=============== Test result diff =================\n";
+  int32_t res_label_expect = 0;
+  float res_score_expect = 1.0;
+  // Calculate diff between two results.
+  auto ppocr_cls_label_diff = res_label - res_label_expect;
+  auto ppocr_cls_score_diff = res_score - res_score_expect;
+  std::cout << "PPOCR Cls label diff: " << ppocr_cls_label_diff << std::endl;
+  std::cout << "PPOCR Cls score diff: " << abs(ppocr_cls_score_diff)
+            << std::endl;
+  BENCHMARK_MODEL(model_ppocr_cls,
+                  model_ppocr_cls.Predict(im, &res_label, &res_score));
+#endif
+  return 0;
+}