Integrate FlagCX into Paddle

.gitmodules

@@ -129,3 +129,7 @@
 	path = third_party/openvino
 	url = https://github.com/openvinotoolkit/openvino.git
 	ignore = dirty
+[submodule "third_party/flagcx"]
+	path = third_party/flagcx
+	url = https://github.com/FlagOpen/FlagCX.git
+	ignore = dirty

CMakeLists.txt

@@ -313,6 +313,7 @@ option(
   OFF)
 option(WITH_CINN "Compile PaddlePaddle with CINN" OFF)
 option(WITH_NCCL "Compile PaddlePaddle with NCCL support" ON)
+option(WITH_FLAGCX "Compile PaddlePaddle with FLAGCX support" OFF)
 option(WITH_RCCL "Compile PaddlePaddle with RCCL support" ON)
 option(WITH_XPU_BKCL "Compile PaddlePaddle with BAIDU KUNLUN XPU BKCL" OFF)
 option(WITH_CRYPTO "Compile PaddlePaddle with crypto support" ON)
@@ -538,6 +539,11 @@ else()
   endif()
 endif()
 
+if(WITH_FLAGCX)
+  add_definitions("-DPADDLE_WITH_FLAGCX")
+  # include(flagcx)
+endif()
+
 if(WITH_HETERPS AND WITH_PSLIB)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
 endif()

cmake/external/flagcx.cmake

@@ -0,0 +1,47 @@
+set(CMAKE_FIND_DEBUG_MODE ON)
+# flagcx.cmake
+if(NOT WITH_FLAGCX)
+  return()
+endif()
+
+set(FLAGCX_SOURCE_DIR "${PADDLE_SOURCE_DIR}/third_party/flagcx")
+set(FLAGCX_BINARY_DIR "${PADDLE_SOURCE_DIR}/build/third_party/flagcx")
+set(THIRD_PARTY_DIR "${PADDLE_SOURCE_DIR}/build/third_party")
+set(FLAGCX_ROOT "/usr/local/flagcx")
+set(FLAGCX_LIB_DIR "${FLAGCX_BINARY_DIR}/build/lib")
+set(USR_LOCAL_DIR "/usr/local")
+
+file(REMOVE_RECURSE ${FLAGCX_BINARY_DIR})
+message(STATUS "removed old flagcx dir")
+message(STATUS "Copying third-party source to build directory")
+execute_process(COMMAND cp -r ${FLAGCX_SOURCE_DIR} ${THIRD_PARTY_DIR}
+                RESULT_VARIABLE COPY_RESULT)
+
+if(NOT COPY_RESULT EQUAL 0)
+  message(FATAL_ERROR "Failed to copy third-party source to build directory")
+endif()
+
+# Create a custom target to build the third-party library
+message(STATUS "Building third-party library with its Makefile")
+execute_process(
+  COMMAND make
+  WORKING_DIRECTORY ${FLAGCX_BINARY_DIR}
+  RESULT_VARIABLE BUILD_RESULT)
+
+find_path(
+  FLAGCX_INCLUDE_DIR flagcx.h
+  PATHS ${FLAGCX_SOURCE_DIR}/flagcx/include
+  NO_DEFAULT_PATH)
+
+message(STATUS "FLAGCX_INCLUDE_DIR is ${FLAGCX_INCLUDE_DIR}")
+include_directories(SYSTEM ${FLAGCX_INCLUDE_DIR})
+
+add_library(flagcx INTERFACE)
+find_library(
+  FLAGCX_LIB
+  NAMES flagcx libflagcx
+  PATHS ${FLAGCX_LIB_DIR}
+  DOC "My custom library")
+
+add_dependencies(flagcx FLAGCX_LIB)
+message(STATUS "FLAGCX_LIB is ${FLAGCX_LIB}")

cmake/third_party.cmake

@@ -469,6 +469,10 @@ if(WITH_TESTING OR WITH_DISTRIBUTE)
   include(external/gtest) # download, build, install gtest
   list(APPEND third_party_deps extern_gtest)
 endif()
+if(WITH_FLAGCX)
+  include(external/flagcx)
+  list(APPEND third_party_deps flagcx)
+endif()
 
 if(WITH_ONNXRUNTIME)
   include(external/onnxruntime

paddle/common/flags.cc

@@ -1265,6 +1265,19 @@ PHI_DEFINE_EXPORTED_bool(multi_node_sample_use_gpu_table,
 PHI_DEFINE_EXPORTED_bool(nccl_blocking_wait, false, "nccl blocking wait");
 #endif
 
+/**
+ * ProcessGroupFlagCX related FLAG
+ * Name: flagcx_blocking_wait
+ * Since Version:
+ * Value Range: bool, default=false
+ * Example:
+ * Note: nccl blocking wait.
+ * blocks host thread until collective operation completes
+ */
+#if defined(PADDLE_WITH_FLAGCX)
+PHI_DEFINE_EXPORTED_bool(flagcx_blocking_wait, false, "flagcx blocking wait");
+#endif
+
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
 PHI_DEFINE_EXPORTED_bool(benchmark_nccl,
                          false,
@@ -1770,6 +1783,13 @@ PHI_DEFINE_EXPORTED_string(
     "For instance, /usr/local/cuda/lib64. If default, "
     "dlopen will search cuda from LD_LIBRARY_PATH");
 
+PHI_DEFINE_EXPORTED_string(
+    flagcx_dir,  // NOLINT
+    "/usr/local/flagcx/build/lib",
+    "Specify path for loading libflagcx.so. For instance, "
+    "For instance, /usr/local/flagcx/lib. If default, "
+    "dlopen will search flagcx from LD_LIBRARY_PATH");
+
 PHI_DEFINE_EXPORTED_string(cupti_dir,
                            "",
                            "Specify path for loading cupti.so.");  // NOLINT

paddle/fluid/distributed/collective/CMakeLists.txt

@@ -36,6 +36,13 @@ if(WITH_NCCL OR WITH_RCCL)
 
 endif()
 
+if(WITH_FLAGCX)
+  cc_library(
+    process_group_flagcx
+    SRCS process_group_flagcx.cc common.cc
+    DEPS process_group phi)
+endif()
+
 if(WITH_XPU_BKCL)
   cc_library(
     process_group_bkcl