Merge branch 'PaddlePaddle:develop' into 4090

Author: swgu98

paddle/cinn/backends/nvrtc/nvrtc_util.cc

@@ -183,7 +183,14 @@ std::string Compiler::CompileCudaSource(const std::string& code,
       nvrtcCompileProgram(prog, param_cstrings.size(), param_cstrings.data());
 
   if (compile_res != NVRTC_SUCCESS) {
-    std::string new_code = CodeGenCudaDev::GetGeneralSourceHeader() + code;
+    std::string new_code = code;
+    std::string from = CodeGenCudaDev::GetSourceHeader();
+    size_t pos = new_code.find(from);
+    if (pos != std::string::npos) {
+      new_code.replace(
+          pos, from.length(), CodeGenCudaDev::GetGeneralSourceHeader());
+    }
+
     NVRTC_CALL(nvrtcCreateProgram(&prog,
                                   new_code.c_str(),
                                   nullptr,
@@ -193,6 +200,7 @@ std::string Compiler::CompileCudaSource(const std::string& code,
     compile_res =
         nvrtcCompileProgram(prog, param_cstrings.size(), param_cstrings.data());
   }
+
   {  // get log
     size_t log_size;
     NVRTC_CALL(nvrtcGetProgramLogSize(prog, &log_size));

paddle/common/flags.cc

@@ -1265,6 +1265,19 @@ PHI_DEFINE_EXPORTED_bool(multi_node_sample_use_gpu_table,
 PHI_DEFINE_EXPORTED_bool(nccl_blocking_wait, false, "nccl blocking wait");
 #endif
 
+/**
+ * ProcessGroupFlagCX related FLAG
+ * Name: flagcx_blocking_wait
+ * Since Version:
+ * Value Range: bool, default=false
+ * Example:
+ * Note: nccl blocking wait.
+ * blocks host thread until collective operation completes
+ */
+#if defined(PADDLE_WITH_FLAGCX)
+PHI_DEFINE_EXPORTED_bool(flagcx_blocking_wait, false, "flagcx blocking wait");
+#endif
+
 #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
 PHI_DEFINE_EXPORTED_bool(benchmark_nccl,
                          false,

paddle/fluid/distributed/collective/CMakeLists.txt

@@ -36,6 +36,13 @@ if(WITH_NCCL OR WITH_RCCL)
 
 endif()
 
+if(WITH_FLAGCX)
+  cc_library(
+    process_group_flagcx
+    SRCS process_group_flagcx.cc common.cc
+    DEPS process_group phi)
+endif()
+
 if(WITH_XPU_BKCL)
   cc_library(
     process_group_bkcl