Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into rewrite_multi_gru

Author: paulinagacek

CMakeLists.txt

@@ -315,6 +315,7 @@ option(WITH_CUDNN_FRONTEND
        "Compile with CUDNN Frontend API support (experimental)" OFF)
 option(WITH_CUDNN_DSO "Compile PaddlePaddle with cuDNN dynamic-link libraries"
        OFF)
+option(WITH_NEWIR "Compile PaddlePaddle with NEWIR" OFF)
 
 if(WITH_RECORD_BUILDTIME)
   set_property(

cmake/external/cinn.cmake

@@ -17,8 +17,8 @@ if(NOT WITH_CINN)
 endif()
 
 if(NOT CINN_GIT_TAG)
-  # 2023.01.28 commit
-  set(CINN_GIT_TAG 1449890f7724babf2a343c6f8073bd28a7bbc683)
+  # 2023.02.09 commit
+  set(CINN_GIT_TAG 63cf4267a0f8756df61b0dca5fb30bf3748ba2f4)
 endif()
 
 message(STATUS "CINN version: " ${CINN_GIT_TAG})

cmake/external/leveldb.cmake

@@ -23,7 +23,10 @@ set(LEVELDB_LIBRARIES
     "${LEVELDB_INSTALL_DIR}/lib/libleveldb.a"
     CACHE FILEPATH "leveldb library." FORCE)
 include_directories(${LEVELDB_INCLUDE_DIR})
-
+set(LEVELDN_CXXFLAGS "-fPIC")
+if(WITH_HETERPS AND WITH_PSLIB)
+  set(LEVELDN_CXXFLAGS "${LEVELDN_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
+endif()
 ExternalProject_Add(
   extern_leveldb
   ${EXTERNAL_PROJECT_LOG_ARGS}
@@ -32,15 +35,15 @@ ExternalProject_Add(
   GIT_TAG v1.18
   UPDATE_COMMAND ""
   CONFIGURE_COMMAND ""
-  BUILD_COMMAND CXXFLAGS=-fPIC make -j ${NUM_OF_PROCESSOR} libleveldb.a
+  BUILD_COMMAND export "CXXFLAGS=${LEVELDN_CXXFLAGS}" && make -j
+                ${NUM_OF_PROCESSOR} libleveldb.a
   INSTALL_COMMAND
     mkdir -p ${LEVELDB_INSTALL_DIR}/lib/ && cp
     ${LEVELDB_PREFIX_DIR}/src/extern_leveldb/libleveldb.a ${LEVELDB_LIBRARIES}
     && cp -r ${LEVELDB_PREFIX_DIR}/src/extern_leveldb/include
     ${LEVELDB_INSTALL_DIR}/
   BUILD_IN_SOURCE 1
   BUILD_BYPRODUCTS ${LEVELDB_LIBRARIES})
-
 add_dependencies(extern_leveldb snappy)
 
 add_library(leveldb STATIC IMPORTED GLOBAL)

cmake/external/lite.cmake

@@ -120,7 +120,11 @@ if(NOT LITE_SOURCE_DIR OR NOT LITE_BINARY_DIR)
         ${LITE_PREFIX_DIR}/src/extern_lite/cmake/os/armlinux.cmake
       UPDATE_COMMAND ""
       BUILD_COMMAND ${LITE_BUILD_COMMAND}
-      INSTALL_COMMAND ""
+      INSTALL_COMMAND
+        cp ${PADDLE_BINARY_DIR}/paddle/fluid/framework/framework.pb.cc
+        ${LITE_PREFIX_DIR}/src/extern_lite-build/lite/core/ && cp
+        ${PADDLE_BINARY_DIR}/paddle/fluid/framework/framework.pb.h
+        ${LITE_PREFIX_DIR}/src/extern_lite-build/lite/core/
       CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
                  -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
                  -DCMAKE_CXX_FLAGS=${LITE_CMAKE_CXX_FLAGS}
@@ -169,7 +173,11 @@ if(NOT LITE_SOURCE_DIR OR NOT LITE_BINARY_DIR)
         "s?NNadapter_bridges_path = os.path.abspath('..')+\"\/lite\/kernels\/nnadapter\/bridges\/paddle_use_bridges.h\"?NNadapter_bridges_path = os.path.abspath(\'..\')+\"\/extern_lite\/lite\/kernels\/nnadapter\/bridges\/paddle_use_bridges.h\"?"
         ${LITE_PREFIX_DIR}/src/extern_lite//lite/tools/cmake_tools/record_supported_kernel_op.py
       BUILD_COMMAND ${LITE_BUILD_COMMAND}
-      INSTALL_COMMAND ""
+      INSTALL_COMMAND
+        cp ${PADDLE_BINARY_DIR}/paddle/fluid/framework/framework.pb.cc
+        ${LITE_PREFIX_DIR}/src/extern_lite-build/lite/core/ && cp
+        ${PADDLE_BINARY_DIR}/paddle/fluid/framework/framework.pb.h
+        ${LITE_PREFIX_DIR}/src/extern_lite-build/lite/core/
       CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
                  -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
                  -DCMAKE_CXX_FLAGS=${LITE_CMAKE_CXX_FLAGS}

cmake/external/protobuf.cmake

@@ -75,8 +75,8 @@ function(protobuf_generate_python SRCS)
     list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}_pb2.py")
     add_custom_command(
       OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}_pb2.py"
-      COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} --python_out
-              ${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL}
+      COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} --python_out ${PADDLE_BINARY_DIR}
+              -I${PADDLE_SOURCE_DIR} ${ABS_FIL}
       DEPENDS ${ABS_FIL} ${PROTOBUF_PROTOC_EXECUTABLE}
       COMMENT "Running Python protocol buffer compiler on ${FIL}"
       VERBATIM)
@@ -236,24 +236,24 @@ function(build_protobuf TARGET_NAME BUILD_FOR_HOST)
 
   if(WITH_ASCEND AND NOT WITH_ASCEND_CXX11)
     set(PROTOBUF_REPOSITORY https://gitee.com/tianjianhe/protobuf.git)
-    set(PROTOBUF_TAG v3.8.0)
+    set(PROTOBUF_TAG v21.12)
   elseif(WITH_ASCEND_CL AND NOT WITH_ASCEND_CXX11)
     set(PROTOBUF_REPOSITORY https://gitee.com/tianjianhe/protobuf.git)
-    set(PROTOBUF_TAG v3.8.0)
+    set(PROTOBUF_TAG v21.12)
   elseif(WITH_IPU)
     set(PROTOBUF_REPOSITORY ${GIT_URL}/protocolbuffers/protobuf.git)
-    set(PROTOBUF_TAG d750fbf648256c7c631f51ffdbf67d7c18b0114e)
+    set(PROTOBUF_TAG v21.12)
   elseif(WIN32)
     set(PROTOBUF_REPOSITORY ${GIT_URL}/protocolbuffers/protobuf.git)
     # Change the tag to support building with vs2019
     set(PROTOBUF_TAG 01a05a53f40ca2ac5f0af10c6cc0810bee39b792)
   else()
     set(PROTOBUF_REPOSITORY ${GIT_URL}/protocolbuffers/protobuf.git)
-    set(PROTOBUF_TAG 9f75c5aa851cd877fb0d93ccc31b8567a6706546)
+    set(PROTOBUF_TAG v21.12)
     if(WITH_GPU)
       if(${CMAKE_CUDA_COMPILER_VERSION} LESS 12.0
          AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 12.0)
-        set(PROTOBUF_TAG 2dc747c574b68a808ea4699d26942c8132fe2b09)
+        set(PROTOBUF_TAG v21.12)
       endif()
     endif()
   endif()
@@ -321,17 +321,17 @@ function(build_protobuf TARGET_NAME BUILD_FOR_HOST)
 endfunction()
 
 if(WITH_ASCEND OR WITH_ASCEND_CL)
-  set(PROTOBUF_VERSION 3.8.0)
+  set(PROTOBUF_VERSION 21.12)
 elseif(WITH_IPU)
-  set(PROTOBUF_VERSION 3.6.1)
+  set(PROTOBUF_VERSION 21.12)
 elseif(WITH_ARM_BRPC)
-  set(PROTOBUF_VERSION 3.7.1-baidu-ee-common)
+  set(PROTOBUF_VERSION 21.12-baidu-ee-common)
 else()
-  set(PROTOBUF_VERSION 3.1.0)
+  set(PROTOBUF_VERSION 21.12)
   if(WITH_GPU)
     if(${CMAKE_CUDA_COMPILER_VERSION} LESS 12.0
        AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER 12.0)
-      set(PROTOBUF_VERSION 3.16.0)
+      set(PROTOBUF_VERSION 21.12)
     endif()
   endif()
 endif()

cmake/generic.cmake

@@ -1063,8 +1063,8 @@ function(paddle_protobuf_generate_cpp SRCS HDRS)
     add_custom_command(
       OUTPUT "${_protobuf_protoc_src}" "${_protobuf_protoc_hdr}"
       COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_CURRENT_BINARY_DIR}"
-      COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} -I${CMAKE_CURRENT_SOURCE_DIR}
-              --cpp_out "${CMAKE_CURRENT_BINARY_DIR}" ${ABS_FIL}
+      COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} -I${PADDLE_SOURCE_DIR} --cpp_out
+              "${PADDLE_BINARY_DIR}" ${ABS_FIL}
       # Set `EXTERN_PROTOBUF_DEPEND` only if need to compile `protoc.exe`.
       DEPENDS ${ABS_FIL} ${EXTERN_PROTOBUF_DEPEND}
       COMMENT "Running C++ protocol buffer compiler on ${FIL}"

cmake/operators.cmake

@@ -28,11 +28,7 @@ endfunction()
 
 function(find_phi_register FILENAME ADD_PATH PATTERN)
   # set op_name to OUTPUT
-  set(options "")
-  set(oneValueArgs "")
-  set(multiValueArgs "")
   file(READ ${FILENAME} CONTENT)
-
   string(
     REGEX
       MATCH
@@ -402,6 +398,7 @@ function(op_library TARGET)
     set(op_name "")
     # Add PHI Kernel Registry Message
     find_phi_register(${cc_src} ${pybind_file} "PD_REGISTER_KERNEL")
+    find_phi_register(${cc_src} ${pybind_file} "PD_REGISTER_STRUCT_KERNEL")
     find_phi_register(${cc_src} ${pybind_file} "PD_REGISTER_GENERAL_KERNEL")
     find_register(${cc_src} "REGISTER_OPERATOR" op_name)
     if(NOT ${op_name} EQUAL "")
@@ -453,6 +450,7 @@ function(op_library TARGET)
     set(op_name "")
     # Add PHI Kernel Registry Message
     find_phi_register(${cu_src} ${pybind_file} "PD_REGISTER_KERNEL")
+    find_phi_register(${cu_src} ${pybind_file} "PD_REGISTER_STRUCT_KERNEL")
     find_phi_register(${cu_src} ${pybind_file} "PD_REGISTER_GENERAL_KERNEL")
     find_register(${cu_src} "REGISTER_OP_CUDA_KERNEL" op_name)
     if(NOT ${op_name} EQUAL "")

paddle/CMakeLists.txt

@@ -13,6 +13,8 @@ add_subdirectory(phi)
 add_subdirectory(infrt)
 add_subdirectory(fluid)
 
+add_subdirectory(ir)
+
 # NOTE(zhiqiu): The changes of cc tests
 # Before, (1) the source file of cc tests are distributed in different sub-directories,
 # (2) the tests are added and configured by calling `cc_test()` in each `CMakeLists.txt`,

paddle/fluid/distributed/collective/process_group_custom.cc

@@ -153,12 +153,10 @@ void ProcessGroupCustom::CreateCustomManagerCache(
   std::vector<std::unique_ptr<CustomDeviceContext>> dev_ctx;
   dev_ctx.resize(places.size());
 
-  std::unique_ptr<phi::ccl::CCLComm> comms(
-      new phi::ccl::CCLComm[places.size()]);
   for (size_t i = 0; i < places.size(); ++i) {
     phi::DeviceGuard guard(places[i]);
     ccl_comms[i] = CustomCCLCommManager::Create(
-        device_type, GetSize(), GetRank(), &ccl_id, comms.get() + i);
+        device_type, GetSize(), GetRank(), &ccl_id, new phi::ccl::CCLComm);
     dev_ctx[i].reset(new CustomDeviceContext(places[i]));
   }
 

paddle/fluid/distributed/collective/reducer.cc

@@ -1113,7 +1113,7 @@ void EagerReducer::AllReduceSparse(EagerGroup *group,
   const auto &rank_ = process_group_->GetRank();
   const auto &size_ = process_group_->GetSize();
 
-  framework::Vector<int64_t> rows_num_vector(size_);
+  phi::Vector<int64_t> rows_num_vector(size_);
   rows_num_vector[rank_] = static_cast<int64_t>(src_rows.size());
 
   Tensor rows_num_tensor = paddle::experimental::empty(
@@ -1183,7 +1183,7 @@ void EagerReducer::AllReduceSparse(EagerGroup *group,
     }
     process_group_->AllGather(in, out)->Synchronize();
 
-    framework::Vector<int64_t> dst_rows_vector(rows_num, 0);
+    phi::Vector<int64_t> dst_rows_vector(rows_num, 0);
     auto *dst_rows_dense_tensor =
         std::dynamic_pointer_cast<phi::DenseTensor>(dst_rows_tensor.impl())
             .get();
@@ -1262,7 +1262,7 @@ void EagerReducer::AllReduceSparse(EagerGroup *group,
 
     Tensor dst_rows_tensor =
         paddle::experimental::concat(rows_tensors, phi::Scalar(0));
-    framework::Vector<int64_t> dst_rows_vector(rows_num, 0);
+    phi::Vector<int64_t> dst_rows_vector(rows_num, 0);
     auto *dst_rows_dense_tensor =
         std::dynamic_pointer_cast<phi::DenseTensor>(dst_rows_tensor.impl())
             .get();

paddle/fluid/distributed/fleet_executor/CMakeLists.txt

@@ -6,7 +6,7 @@ proto_library(interceptor_message_proto SRCS interceptor_message.proto)
 
 if(WITH_ARM_BRPC)
   set(BRPC_DEPS arm_brpc snappy gflags glog)
-elseif(WITH_DISTRIBUTE)
+elseif(WITH_DISTRIBUTE AND NOT WITH_PSLIB)
   set(BRPC_DEPS
       brpc
       ssl

paddle/fluid/distributed/fleet_executor/dist_model.cc

@@ -433,10 +433,15 @@ bool DistModel::PrepareFleetExe() {
   executor_desc_ = FleetExecutorDesc();
   executor_desc_.set_cur_rank(config_.local_rank);
   std::unordered_map<int64_t, int64_t> id_to_rank;
+
   for (int i = 0; i < config_.nranks; ++i) {
     RankInfo *rank_info = executor_desc_.add_cluster_info();
     rank_info->set_rank(i);
-    rank_info->set_ip_port(config_.trainer_endpoints[i]);
+    if (config_.nranks == 1) {
+      rank_info->set_ip_port("");
+    } else {
+      rank_info->set_ip_port(config_.trainer_endpoints[i]);
+    }
     id_to_rank.insert({i, i});
   }
   fleet_exe.reset(new FleetExecutor(executor_desc_));