Skip to content

Commit b2fc8f6

Browse files
yeliang2258yeliang2258
authored
Merge branch 'develop' into fix_box_coder_dev
2 parents 2c94a53 + 9a89ef3 commit b2fc8f6

File tree

139 files changed

+10889
-4
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

139 files changed

+10889
-4
lines changed

‎.gitmodules

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
[submodule "third/onnx"]
2+
path = third/onnx
3+
url = https://github.com/onnx/onnx.git
4+
branch = da889e6b95750350726d149bf447bf0cd1245964
5+
[submodule "third/optimizer"]
6+
path = third/optimizer
7+
url = https://github.com/onnx/optimizer.git
8+
branch = a37748b2c3a80dad4274401c45c5026c7a506730
9+
[submodule "third/pybind11"]
10+
path = third/pybind11
11+
url = https://github.com/pybind/pybind11.git
12+
branch = master

‎CMakeLists.txt

+162
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
PROJECT(paddle2onnx C CXX)
2+
CMAKE_MINIMUM_REQUIRED (VERSION 3.16)
3+
4+
ADD_SUBDIRECTORY(paddle2onnx)
5+
6+
option(WITH_STATIC "Compile Paddle2ONNX with STATIC" OFF)
7+
set(ONNX_CUSTOM_PROTOC_PATH "" CACHE STRING "Set yourself Protobufc path")
8+
9+
# Set C++11 as standard for the whole project
10+
if(NOT MSVC)
11+
set(CMAKE_CXX_STANDARD 11)
12+
endif(NOT MSVC)
13+
14+
# setting max opset version for onnx
15+
# if you build from other version of onnx
16+
# this should be modified
17+
# refer to https://github.com/onnx/onnx/blob/main/docs/Versioning.md#released-versions
18+
add_definitions(-DMAX_ONNX_OPSET_VERSION=15)
19+
add_definitions(-DPADDLE2ONNX_LIB)
20+
21+
# Third dependency: onnx
22+
if(NOT TARGET onnx_proto)
23+
if(NOT ONNX_NAMESPACE)
24+
set(ONNX_NAMESPACE "paddle2onnx")
25+
endif()
26+
add_definitions("-DONNX_NAMESPACE=${ONNX_NAMESPACE}")
27+
28+
if(ONNX_CUSTOM_PROTOC_PATH)
29+
if(WIN32)
30+
set(ONNX_USE_MSVC_STATIC_RUNTIME ON)
31+
set(ONNX_CUSTOM_PROTOC_PATH "${ONNX_CUSTOM_PROTOC_PATH};$ENV{PATH}")
32+
else()
33+
set(ONNX_CUSTOM_PROTOC_PATH "${ONNX_CUSTOM_PROTOC_PATH}:$ENV{PATH}")
34+
endif()
35+
set(ENV{PATH} ${ONNX_CUSTOM_PROTOC_PATH})
36+
endif()
37+
38+
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
39+
add_subdirectory(${PROJECT_SOURCE_DIR}/third/onnx)
40+
endif()
41+
42+
include_directories(${PROJECT_SOURCE_DIR})
43+
include_directories(${CMAKE_CURRENT_BINARY_DIR})
44+
include_directories(${PROJECT_SOURCE_DIR}/third/optimizer)
45+
add_subdirectory(${PROJECT_SOURCE_DIR}/paddle2onnx/proto)
46+
47+
file(GLOB_RECURSE ALL_SRCS ${PROJECT_SOURCE_DIR}/paddle2onnx/*.cc ${PROJECT_SOURCE_DIR}/third/optimizer/onnxoptimizer/*.cc)
48+
list(REMOVE_ITEM ALL_SRCS ${PROJECT_SOURCE_DIR}/paddle2onnx/cpp2py_export.cc)
49+
list(REMOVE_ITEM ALL_SRCS ${PROJECT_SOURCE_DIR}/third/optimizer/onnxoptimizer/cpp2py_export.cc)
50+
51+
include_directories(${CMAKE_CURRENT_BINARY_DIR})
52+
if (WITH_STATIC)
53+
ADD_LIBRARY(paddle2onnx STATIC ${ALL_SRCS})
54+
TARGET_LINK_LIBRARIES(paddle2onnx p2o_paddle_proto onnx )
55+
else ()
56+
ADD_LIBRARY(paddle2onnx SHARED ${ALL_SRCS})
57+
TARGET_LINK_LIBRARIES(paddle2onnx p2o_paddle_proto onnx)
58+
endif()
59+
60+
install(
61+
DIRECTORY ${PROJECT_SOURCE_DIR}/paddle2onnx
62+
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
63+
FILES_MATCHING
64+
PATTERN "converter.h"
65+
)
66+
install(
67+
TARGETS paddle2onnx
68+
DESTINATION ${CMAKE_INSTALL_LIBDIR}
69+
)
70+
71+
if (BUILD_PADDLE2ONNX_EXE)
72+
remove_definitions(-DPADDLE2ONNX_LIB)
73+
ADD_EXECUTABLE(p2o_exec p2o_exec.cpp)
74+
if (WITH_STATIC)
75+
TARGET_LINK_LIBRARIES(p2o_exec -Wl,--whole-archive paddle2onnx -Wl,--no-whole-archive)
76+
else()
77+
TARGET_LINK_LIBRARIES(p2o_exec paddle2onnx)
78+
endif()
79+
endif (BUILD_PADDLE2ONNX_EXE)
80+
81+
if(BUILD_PADDLE2ONNX_PYTHON)
82+
if("${PY_EXT_SUFFIX}" STREQUAL "")
83+
if(MSVC)
84+
set(PY_EXT_SUFFIX ".pyd")
85+
else()
86+
set(PY_EXT_SUFFIX ".so")
87+
endif()
88+
endif()
89+
90+
add_library(paddle2onnx_cpp2py_export MODULE ${PROJECT_SOURCE_DIR}/paddle2onnx/cpp2py_export.cc ${ALL_SRCS})
91+
set_target_properties(paddle2onnx_cpp2py_export PROPERTIES PREFIX "")
92+
set_target_properties(paddle2onnx_cpp2py_export
93+
PROPERTIES COMPILE_FLAGS "-fvisibility=hidden")
94+
set_target_properties(paddle2onnx_cpp2py_export PROPERTIES SUFFIX ${PY_EXT_SUFFIX})
95+
set_target_properties(paddle2onnx_cpp2py_export
96+
PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
97+
target_include_directories(paddle2onnx_cpp2py_export PRIVATE
98+
$<BUILD_INTERFACE:${ONNX_ROOT}>
99+
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
100+
$<INSTALL_INTERFACE:include>
101+
${PYTHON_INCLUDE_DIR})
102+
103+
# pybind11 is a header only lib
104+
find_package(pybind11 2.2)
105+
if(pybind11_FOUND)
106+
target_include_directories(paddle2onnx_cpp2py_export PUBLIC
107+
${pybind11_INCLUDE_DIRS})
108+
else()
109+
if(EXISTS ${PROJECT_SOURCE_DIR}/third/pybind11/include/pybind11/pybind11.h)
110+
target_include_directories(paddle2onnx_cpp2py_export PUBLIC
111+
${PROJECT_SOURCE_DIR}/third/pybind11/include)
112+
else()
113+
message(FATAL_ERROR "cannot find pybind")
114+
endif()
115+
endif()
116+
117+
if(APPLE)
118+
set_target_properties(paddle2onnx_cpp2py_export
119+
PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
120+
target_link_libraries(paddle2onnx_cpp2py_export
121+
PRIVATE -Wl,-force_load,$<TARGET_FILE:onnx>)
122+
elseif(MSVC)
123+
# In MSVC, we will add whole archive in default
124+
target_link_libraries(paddle2onnx_cpp2py_export
125+
PRIVATE -WHOLEARCHIVE:$<TARGET_FILE:onnx>)
126+
elseif(CMAKE_SYSTEM_NAME STREQUAL "AIX")
127+
# whole-archive linker option not available on AIX
128+
target_sources(paddle2onnx_cpp2py_export
129+
PRIVATE $<TARGET_OBJECTS:onnx>)
130+
else()
131+
# Assume everything else is like gcc
132+
target_link_libraries(paddle2onnx_cpp2py_export
133+
PRIVATE "-Wl,--whole-archive" $<TARGET_FILE:onnx>
134+
"-Wl,--no-whole-archive")
135+
set_target_properties(paddle2onnx_cpp2py_export
136+
PROPERTIES LINK_FLAGS "-Wl,--exclude-libs,ALL")
137+
endif()
138+
139+
target_link_libraries(paddle2onnx_cpp2py_export PRIVATE p2o_paddle_proto onnx)
140+
141+
if(MSVC)
142+
target_link_libraries(paddle2onnx_cpp2py_export PRIVATE ${PYTHON_LIBRARIES})
143+
target_compile_options(paddle2onnx_cpp2py_export
144+
PRIVATE /MP
145+
/wd4244 # 'argument': conversion from 'google::
146+
# protobuf::uint64' to 'int', possible
147+
# loss of data
148+
/wd4267 # Conversion from 'size_t' to 'int',
149+
# possible loss of data
150+
/wd4996 # The second parameter is ignored.
151+
${EXTRA_FLAGS})
152+
if(ONNX_USE_PROTOBUF_SHARED_LIBS)
153+
target_compile_options(onnx_cpp2py_export
154+
PRIVATE /wd4251 # 'identifier' : class 'type1' needs to
155+
# have dll-interface to be used by
156+
# clients of class 'type2'
157+
)
158+
endif()
159+
add_msvc_runtime_flag(paddle2onnx_cpp2py_export)
160+
add_onnx_global_defines(paddle2onnx_cpp2py_export)
161+
endif()
162+
endif()

‎README.md

+1
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ If you need to configure the input shape, use the following command:
6464
|--params_filename |**[Optional]** the parameter file name under the directory designated by`--model_dir`. Only needed when all the model parameters saved in one binary file. Default value None|
6565
|--save_file | the directory path for the exported ONNX model|
6666
|--opset_version | **[Optional]** To configure the ONNX Opset version. Opset 9-11 are stably supported. Default value is 9.|
67+
|--enable_dev_version | **[Optional]** Whether to use new version of Paddle2ONNX while is under developing. Default value is False.|
6768
|--enable_onnx_checker| **[Optional]** To check the validity of the exported ONNX model. It is suggested to turn on the switch. If set to True, onnx>=1.7.0 is required. Default value is False|
6869
|--enable_paddle_fallback| **[Optional]** Whether custom op is exported using paddle_fallback mode. Default value is False|
6970
|--enable_auto_update_opset| **[Optional]** Whether enable auto_update_opset. Default value is True|

‎README_zh.md

+1
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ Paddle模型的参数保存在一个单独的二进制文件中（combined）:
5959
|--params_filename |**[可选]** 配置位于`--model_dir`下存储模型参数的文件名称。当且仅当所有模型参数被保存在一个单独的二进制文件中，它才需要被指定。默认为None|
6060
|--save_file | 指定转换后的模型保存目录路径 |
6161
|--opset_version | **[可选]** 配置转换为ONNX的OpSet版本，目前比较稳定地支持9、10、11三个版本，默认为9 |
62+
|--enable_dev_version | **[可选]** 是否使用新版本Paddle2ONNX（当前正在开发中），默认为False |
6263
|--enable_onnx_checker| **[可选]** 配置是否检查导出为ONNX模型的正确性, 建议打开此开关。若指定为True，需要安装 onnx>=1.7.0, 默认为False|
6364
|--enable_paddle_fallback| **[可选]** 配置custom op是否使用paddle_fallback模式导出, 默认为False|
6465
|--enable_auto_update_opset| **[可选]** 配置是否开启opset version自动校正功能, 默认为True|

‎VERSION_NUMBER

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
0.9.3

‎paddle2onnx/CMakeLists.txt

Whitespace-only changes.

‎paddle2onnx/__init__.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
# limitations under the License.
1414
from __future__ import absolute_import
1515

16-
__version__ = "0.9.2"
16+
__version__ = "0.9.3"
1717

1818
import paddle
1919
from .convert import dygraph2onnx, program2onnx
@@ -33,12 +33,14 @@
3333
'copy_cross_scope'
3434
}
3535

36+
3637
def process_old_ops_desc(model):
3738
for i in range(len(model.blocks[0].ops)):
3839
if model.blocks[0].ops[i].type == "matmul":
3940
if not model.blocks[0].ops[i].has_attr("head_number"):
4041
model.blocks[0].ops[i]._set_attr("head_number", 1)
4142

43+
4244
def get_all_registered_ops(save_file=None):
4345
ops = list(OpMapper.OPSETS.keys())
4446
logging.warning("The number of all registered OPs is: {}".format(len(ops)))

‎paddle2onnx/command.py

+52-2
Original file line numberDiff line numberDiff line change
@@ -72,12 +72,17 @@ def arg_parser():
7272
default="None",
7373
help="define input shapes, e.g --input_shape_dict=\"{'image':[1, 3, 608, 608]}\" or" \
7474
"--input_shape_dict=\"{'image':[1, 3, 608, 608], 'im_shape': [1, 2], 'scale_factor': [1, 2]}\"")
75+
parser.add_argument(
76+
"--enable_dev_version",
77+
type=ast.literal_eval,
78+
default=False,
79+
help="whether to use new version of Paddle2ONNX which is under developing, default False"
80+
)
7581
parser.add_argument(
7682
"--enable_onnx_checker",
7783
type=ast.literal_eval,
7884
default=True,
79-
help="whether check onnx model validity, if True, please 'pip install onnx'"
80-
)
85+
help="whether check onnx model validity, default True")
8186
parser.add_argument(
8287
"--enable_paddle_fallback",
8388
type=ast.literal_eval,
@@ -105,6 +110,26 @@ def arg_parser():
105110
return parser
106111

107112

113+
def c_paddle_to_onnx(model_file,
114+
params_file="",
115+
save_file=None,
116+
opset_version=7,
117+
auto_upgrade_opset=True,
118+
verbose=True,
119+
enable_onnx_checker=True,
120+
enable_experimental_op=True,
121+
enable_optimize=True):
122+
import paddle2onnx.paddle2onnx_cpp2py_export as c_p2o
123+
onnx_model_str = c_p2o.export(
124+
model_file, params_file, opset_version, auto_upgrade_opset, verbose,
125+
enable_onnx_checker, enable_experimental_op, enable_optimize)
126+
if save_file is not None:
127+
with open(save_file, "wb") as f:
128+
f.write(onnx_model_str)
129+
else:
130+
return onnx_model_str
131+
132+
108133
def program2onnx(model_dir,
109134
save_file,
110135
model_filename=None,
@@ -217,6 +242,31 @@ def main():
217242
"The output_names should be 'list' or 'dict', but received type is %s."
218243
% type(args.output_names))
219244

245+
if args.enable_dev_version:
246+
if args.enable_paddle_fallback:
247+
logging.warn(
248+
"--enable_paddle_fallback is deprecated while --enable_dev_version=True."
249+
)
250+
if args.output_names is not None:
251+
logging.warn(
252+
"--output_names is deprecated while --enable_dev_version=True.")
253+
if input_shape_dict is not None:
254+
logging.warn(
255+
"--input_shape_dict is deprecated while --enable_dev_version=True."
256+
)
257+
model_file = os.path.join(args.model_dir, args.model_filename)
258+
params_file = os.path.join(args.model_dir, args.params_filename)
259+
return c_paddle_to_onnx(
260+
model_file=model_file,
261+
params_file=params_file,
262+
save_file=args.save_file,
263+
opset_version=args.opset_version,
264+
auto_upgrade_opset=args.enable_auto_update_opset,
265+
verbose=True,
266+
enable_onnx_checker=args.enable_onnx_checker,
267+
enable_experimental_op=True,
268+
enable_optimize=True)
269+
220270
program2onnx(
221271
args.model_dir,
222272
args.save_file,

‎paddle2onnx/converter.cc

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#include "paddle2onnx/converter.h"
16+
#include <fstream>
17+
#include <iostream>
18+
#include <set>
19+
#include "paddle2onnx/mapper/exporter.h"
20+
21+
namespace paddle2onnx {
22+
23+
PADDLE2ONNX_DECL bool IsExportable(
24+
const std::string& model, const std::string& params,
25+
bool from_memory_buffer, int32_t opset_version, bool auto_upgrade_opset,
26+
bool verbose, bool enable_onnx_checker, bool enable_experimental_op,
27+
bool enable_optimize) {
28+
auto parser = PaddleParser();
29+
if (!parser.Init(model, params, from_memory_buffer)) {
30+
return false;
31+
}
32+
paddle2onnx::ModelExporter me;
33+
std::set<std::string> unsupported_ops;
34+
if (!me.CheckIfOpSupported(parser, &unsupported_ops,
35+
enable_experimental_op)) {
36+
return false;
37+
}
38+
if (me.GetMinOpset(parser, false) < 0) {
39+
return false;
40+
}
41+
std::string onnx_model =
42+
me.Run(parser, opset_version, auto_upgrade_opset, verbose,
43+
enable_onnx_checker, enable_experimental_op, enable_optimize);
44+
if (onnx_model.empty()) {
45+
P2OLogger(verbose) << "The exported ONNX model is invalid!" << std::endl;
46+
return false;
47+
}
48+
return true;
49+
}
50+
51+
PADDLE2ONNX_DECL bool Export(const std::string& model,
52+
const std::string& params, std::string* out,
53+
bool from_memory_buffer, int32_t opset_version,
54+
bool auto_upgrade_opset, bool verbose,
55+
bool enable_onnx_checker,
56+
bool enable_experimental_op,
57+
bool enable_optimize) {
58+
auto parser = PaddleParser();
59+
P2OLogger(verbose) << "Start to parsing Paddle model..." << std::endl;
60+
if (!parser.Init(model, params, from_memory_buffer)) {
61+
P2OLogger(verbose) << "Paddle model parsing failed." << std::endl;
62+
return false;
63+
}
64+
paddle2onnx::ModelExporter me;
65+
*out = me.Run(parser, opset_version, auto_upgrade_opset, verbose,
66+
enable_onnx_checker, enable_experimental_op, enable_optimize);
67+
if (out->empty()) {
68+
P2OLogger(verbose) << "The exported ONNX model is invalid!" << std::endl;
69+
return false;
70+
}
71+
return true;
72+
}
73+
} // namespace paddle2onnx

0 commit comments

Comments
 (0)