Skip to content

Commit d41ac09

Browse files
reyoungreyoung
authored
Merge pull request #1017 from gangliao/external
Download, build, install External dependencies via cmake
2 parents 0057c80 + d0a5ce2 commit d41ac09

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+1020
-470
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ build/
99
.pydevproject
1010
Makefile
1111
.test_env/
12+
third_party/
1213

1314
*~
1415
bazel-*

‎.gitmodules

-3
This file was deleted.

‎.pre-commit-config.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
sha: c25201a00e6b0514370501050cf2a8538ac12270
33
hooks:
44
- id: remove-crlf
5-
files: (?!.*warp-ctc)^.*$
5+
files: (?!.*third_party)^.*$
66
- repo: https://github.com/reyoung/mirrors-yapf.git
77
sha: v0.13.2
88
hooks:
@@ -15,7 +15,7 @@
1515
- id: check-merge-conflict
1616
- id: check-symlinks
1717
- id: detect-private-key
18-
files: (?!.*warp-ctc)^.*$
18+
files: (?!.*third_party)^.*$
1919
- id: end-of-file-fixer
2020
- repo: https://github.com/PaddlePaddle/clang-format-pre-commit-hook.git
2121
sha: 28c0ea8a67a3e2dbbf4822ef44e85b63a0080a29

‎.travis.yml

+2-10
Original file line numberDiff line numberDiff line change
@@ -28,16 +28,8 @@ addons:
2828
- python
2929
- python-pip
3030
- python2.7-dev
31-
- m4
32-
- python-numpy
33-
- python-wheel
34-
- libgoogle-glog-dev
35-
- libgflags-dev
36-
- libgtest-dev
3731
- curl
38-
- lcov
3932
- graphviz
40-
- swig
4133
- clang-format-3.8
4234
- automake
4335
- libtool
@@ -53,10 +45,10 @@ before_install:
5345
fi
5446
fi
5547
fi
56-
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo paddle/scripts/travis/before_install.linux.sh; fi
5748
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi
5849
- if [[ "$JOB" == "PRE_COMMIT" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi
59-
- pip install wheel protobuf sphinx recommonmark virtualenv numpy sphinx_rtd_theme pre-commit requests==2.9.2 LinkChecker
50+
- pip install --upgrade pip
51+
- pip install wheel protobuf sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker
6052
script:
6153
- paddle/scripts/travis/main.sh
6254
notifications:

‎CMakeLists.txt

+58-135
Original file line numberDiff line numberDiff line change
@@ -1,166 +1,89 @@
1-
cmake_minimum_required(VERSION 2.8)
1+
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License
14+
15+
cmake_minimum_required(VERSION 3.0)
216

317
project(paddle CXX C)
418

519
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake")
620
set(PROJ_ROOT ${CMAKE_SOURCE_DIR})
7-
include(package)
8-
find_package(SWIG 2.0)
9-
find_package(CUDA QUIET)
10-
find_package(Protobuf REQUIRED)
1121

12-
# Check protobuf library version.
13-
execute_process(COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} --version
14-
OUTPUT_VARIABLE PROTOBUF_VERSION)
15-
string(REPLACE "libprotoc " "" PROTOBUF_VERSION ${PROTOBUF_VERSION})
22+
find_package(Sphinx)
23+
find_package(CUDA QUIET)
24+
find_package(Git REQUIRED)
25+
find_package(Threads REQUIRED)
1626

17-
set(PROTOBUF_3 OFF)
18-
if (${PROTOBUF_VERSION} VERSION_GREATER "3.0.0" OR ${PROTOBUF_VERSION} VERSION_EQUAL "3.0.0")
19-
set(PROTOBUF_3 ON)
20-
endif()
27+
include(system)
28+
include(simd)
2129

22-
find_package(PythonLibs 2.7 REQUIRED)
23-
find_package(PythonInterp 2.7 REQUIRED)
24-
find_package(ZLIB REQUIRED)
25-
find_package(NumPy REQUIRED)
26-
find_package(Threads REQUIRED)
27-
find_package(AVX QUIET)
28-
find_package(Glog REQUIRED)
29-
find_package(Gflags REQUIRED)
30-
find_package(GTest)
31-
find_package(Sphinx)
32-
find_package(Doxygen)
33-
include(cblas)
34-
find_program(M4_EXECUTABLE m4)
35-
###################### Configurations ###########################
30+
###################### Configurations ############################
3631
option(WITH_DSO "Compile PaddlePaddle with dynamic linked libraries" ON)
3732
option(WITH_GPU "Compile PaddlePaddle with gpu" ${CUDA_FOUND})
3833
option(WITH_DOUBLE "Compile PaddlePaddle with double precision, otherwise use single precision" OFF)
3934
option(WITH_AVX "Compile PaddlePaddle with avx intrinsics" ${AVX_FOUND})
4035
option(WITH_PYTHON "Compile PaddlePaddle with python interpreter" ON)
41-
option(WITH_STYLE_CHECK "Style Check for PaddlePaddle" ${PYTHONINTERP_FOUND})
36+
option(WITH_STYLE_CHECK "Style Check for PaddlePaddle" ON)
4237
option(WITH_RDMA "Compile PaddlePaddle with rdma support" OFF)
4338
option(WITH_TIMER "Compile PaddlePaddle use timer" OFF)
4439
option(WITH_PROFILER "Compile PaddlePaddle use gpu profiler" OFF)
45-
option(WITH_TESTING "Compile and run unittest for PaddlePaddle" ${GTEST_FOUND})
40+
option(WITH_TESTING "Compile and run unittest for PaddlePaddle" ON)
4641
option(WITH_DOC "Compile PaddlePaddle with documentation" OFF)
47-
option(WITH_SWIG_PY "Compile PaddlePaddle with py PaddlePaddle prediction api" ${SWIG_FOUND})
42+
option(WITH_SWIG_PY "Compile PaddlePaddle with py PaddlePaddle prediction api" ON)
4843
option(ON_TRAVIS "Running test on travis-ci or not." OFF)
4944
option(ON_COVERALLS "Generating code coverage data on coveralls or not." OFF)
5045
option(COVERALLS_UPLOAD "Uploading the generated coveralls json." ON)
5146

47+
include(external/zlib) # download, build, install zlib
48+
include(external/gflags) # download, build, install gflags
49+
include(external/glog) # download, build, install glog
50+
include(external/gtest) # download, build, install gtest
51+
include(external/protobuf) # download, build, install protobuf
52+
include(external/python) # download, build, install python
53+
include(external/openblas) # download, build, install openblas
54+
include(external/swig) # download, build, install swig
55+
include(external/warpctc) # download, build, install warpctc
56+
57+
include(package) # set paddle packages
58+
include(cpplint) # set paddle c++ style
59+
include(ccache) # set ccache for compilation
60+
include(util) # set unittest and link libs
61+
include(rdma) # set rdma libraries
62+
include(flags) # set paddle compile flags
63+
include(cudnn) # set cudnn libraries
64+
include(version) # set PADDLE_VERSION
65+
include(coveralls) # set code coverage
66+
include(python_module) # set python module
67+
68+
include(configure) # add paddle env configuration
5269

53-
include(cpplint)
54-
include(ccache)
55-
if(WITH_RDMA)
56-
include(rdma)
57-
endif()
58-
include(util)
59-
include(flags)
60-
include(cudnn)
61-
include(FindPythonModule)
62-
include(check_packages)
63-
include(swig)
64-
include(coveralls)
65-
66-
# Set PaddlePaddle version to Git tag name or Git commit ID.
67-
find_package(Git REQUIRED)
68-
# version.cmake will get the current PADDLE_VERSION
69-
include(version)
70-
add_definitions(-DPADDLE_VERSION=${PADDLE_VERSION})
71-
72-
if(NOT WITH_GPU)
73-
add_definitions(-DPADDLE_ONLY_CPU)
74-
add_definitions(-DHPPL_STUB_FUNC)
75-
76-
list(APPEND CMAKE_CXX_SOURCE_FILE_EXTENSIONS cu)
77-
else()
78-
if(${CUDA_VERSION_MAJOR} VERSION_LESS 7)
79-
message(FATAL_ERROR "Paddle need CUDA >= 7.0 to compile")
80-
endif()
81-
82-
if(NOT CUDNN_FOUND)
83-
message(FATAL_ERROR "Paddle need cudnn to compile")
84-
endif()
85-
86-
if(WITH_AVX)
87-
set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler ${AVX_FLAG}")
88-
else(WITH_AVX)
89-
set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler ${SSE3_FLAG}")
90-
endif(WITH_AVX)
91-
92-
# Include cuda and cudnn
93-
include_directories(${CUDNN_INCLUDE_DIR})
94-
include_directories(${CUDA_TOOLKIT_INCLUDE})
95-
endif(NOT WITH_GPU)
96-
97-
if(WITH_DSO)
98-
add_definitions(-DPADDLE_USE_DSO)
99-
endif(WITH_DSO)
100-
101-
if(WITH_DOUBLE)
102-
add_definitions(-DPADDLE_TYPE_DOUBLE)
103-
set(ACCURACY double)
104-
else(WITH_DOUBLE)
105-
set(ACCURACY float)
106-
endif(WITH_DOUBLE)
107-
108-
if(NOT WITH_TIMER)
109-
add_definitions(-DPADDLE_DISABLE_TIMER)
110-
endif(NOT WITH_TIMER)
111-
112-
if(NOT WITH_PROFILER)
113-
add_definitions(-DPADDLE_DISABLE_PROFILER)
114-
endif(NOT WITH_PROFILER)
115-
116-
if(WITH_AVX)
117-
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${AVX_FLAG}")
118-
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${AVX_FLAG}")
119-
else(WITH_AVX)
120-
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SSE3_FLAG}")
121-
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SSE3_FLAG}")
122-
endif(WITH_AVX)
123-
124-
if(WITH_PYTHON)
125-
include_directories(${PYTHON_INCLUDE_DIR})
126-
include_directories(${PYTHON_NUMPY_INCLUDE_DIR})
127-
else(WITH_PYTHON)
128-
add_definitions(-DPADDLE_NO_PYTHON)
129-
endif(WITH_PYTHON)
130-
131-
if(WITH_RDMA)
132-
include_directories("${RDMA_INC_DIR}")
133-
else(WITH_RDMA)
134-
add_definitions(-DPADDLE_DISABLE_RDMA)
135-
endif(WITH_RDMA)
136-
137-
# glog
138-
include_directories(${LIBGLOG_INCLUDE_DIR})
139-
140-
#gflags
141-
add_definitions(-DGFLAGS_NS=${GFLAGS_NAMESPACE})
142-
include_directories(${GFLAGS_INCLUDE_DIRS})
143-
144-
if(WITH_TESTING)
145-
enable_testing()
146-
include_directories(${GTEST_INCLUDE_DIRS})
147-
endif()
148-
149-
include_directories("${CBLAS_INC_DIR}")
15070
include_directories("${PROJ_ROOT}")
15171
include_directories("${PROJ_ROOT}/paddle/cuda/include")
152-
include_directories(${PROTOBUF_INCLUDE_DIRS})
15372
include_directories("${CMAKE_CURRENT_BINARY_DIR}/proto")
154-
if(EXISTS "${PROJ_ROOT}/paddle/internals/CMakeLists.txt")
155-
set(PADDLE_WITH_INTERNAL ON)
156-
include(paddle/internals/CMakeLists.txt)
157-
else()
158-
set(PADDLE_WITH_INTERNAL OFF)
159-
set(INTERNAL_PROTO_PATH "")
160-
endif()
73+
74+
set(EXTERNAL_LIBS
75+
# have not include gtest here.
76+
${GFLAGS_LIBRARIES}
77+
${GLOG_LIBRARIES}
78+
${CBLAS_LIBRARIES}
79+
${PROTOBUF_LIBRARY}
80+
${ZLIB_LIBRARIES}
81+
)
82+
16183
add_subdirectory(proto)
16284
add_subdirectory(paddle)
16385
add_subdirectory(python)
86+
16487
if(WITH_DOC)
16588
add_subdirectory(doc)
16689
endif()

‎cmake/cblas.cmake

+13-11
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
# system paths.
1414
#
1515

16+
set(CBLAS_FOUND OFF)
1617

1718
## Find MKL First.
1819
set(MKL_ROOT $ENV{MKL_ROOT} CACHE PATH "Folder contains MKL")
@@ -35,11 +36,12 @@ find_library(MKL_INTEL_LP64 NAMES mkl_intel_lp64 PATHS
3536
if(MKL_INCLUDE_DIR AND MKL_CORE_LIB AND MKL_SEQUENTIAL_LIB AND MKL_INTEL_LP64)
3637
set(CBLAS_PROVIDER MKL)
3738
set(CBLAS_INC_DIR ${MKL_INCLUDE_DIR})
38-
set(CBLAS_LIBS ${MKL_INTEL_LP64}
39+
set(CBLAS_LIBRARIES ${MKL_INTEL_LP64}
3940
${MKL_SEQUENTIAL_LIB}
4041
${MKL_CORE_LIB})
4142
add_definitions(-DPADDLE_USE_MKL)
42-
message(STATUS "Found MKL (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBS})")
43+
message(STATUS "Found MKL (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBRARIES})")
44+
set(CBLAS_FOUND ON)
4345
return() # return file.
4446
endif()
4547

@@ -68,9 +70,10 @@ find_library(ATLAS_LIB NAMES lapack_atlas liblapack_atlas.so.3
6870
if(ATLAS_INC_DIR AND ATLAS_CBLAS_LIB AND ATLAS_LIB)
6971
set(CBLAS_PROVIDER ATLAS)
7072
set(CBLAS_INC_DIR ${ATLAS_INC_DIR} ${ATLAS_CLAPACK_INC_DIR})
71-
set(CBLAS_LIBS ${ATLAS_LIB} ${ATLAS_CBLAS_LIB})
73+
set(CBLAS_LIBRARIES ${ATLAS_LIB} ${ATLAS_CBLAS_LIB})
7274
add_definitions(-DPADDLE_USE_ATLAS)
73-
message(STATUS "Found Atlas (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBS})")
75+
message(STATUS "Found Atlas (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBRARIES})")
76+
set(CBLAS_FOUND ON)
7477
return()
7578
endif()
7679

@@ -98,8 +101,9 @@ find_library(OPENBLAS_LIB NAMES openblas
98101
if(OPENBLAS_INC_DIR AND OPENBLAS_LIB)
99102
set(CBLAS_PROVIDER OPENBLAS)
100103
set(CBLAS_INC_DIR ${OPENBLAS_INC_DIR})
101-
set(CBLAS_LIBS ${OPENBLAS_LIB})
102-
message(STATUS "Found OpenBlas (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBS})")
104+
set(CBLAS_LIBRARIES ${OPENBLAS_LIB})
105+
message(STATUS "Found OpenBlas (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBRARIES})")
106+
set(CBLAS_FOUND ON)
103107
return()
104108
endif()
105109

@@ -130,9 +134,7 @@ find_library(REFERENCE_CBLAS_LIBRARY NAMES cblas PATHS
130134
if (REFERENCE_CBLAS_INCLUDE_DIR AND REFERENCE_CBLAS_LIBRARY)
131135
set(CBLAS_PROVIDER REFERENCE)
132136
set(CBLAS_INC_DIR ${REFERENCE_CBLAS_INCLUDE_DIR})
133-
set(CBLAS_LIBS ${REFERENCE_CBLAS_LIBRARY})
134-
return()
137+
set(CBLAS_LIBRARIES ${REFERENCE_CBLAS_LIBRARY})
138+
message(STATUS "Found reference-cblas (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBS})")
139+
set(CBLAS_FOUND ON)
135140
endif()
136-
137-
message(FATAL_ERROR "CBlas must be set. Paddle support MKL, ATLAS, OpenBlas, reference-cblas."
138-
" Try set MKL_ROOT, ATLAS_ROOT, OPENBLAS_ROOT or REFERENCE_CBLAS_ROOT.")

‎cmake/check_packages.cmake

-39
This file was deleted.

0 commit comments

Comments
 (0)