【dygraph】support python fused/sparse Grad api (#71029)

* grad_api2

* support sparse

* modify dygraph_api

Author: xiaoguoguo626807

paddle/fluid/eager/auto_code_generator/generator/CMakeLists.txt

@@ -4,9 +4,7 @@ set(api_yaml_path
 set(backward_yaml_path
     "${PADDLE_SOURCE_DIR}/paddle/phi/ops/yaml/backward.yaml,${PADDLE_SOURCE_DIR}/paddle/phi/ops/yaml/inconsistent/dygraph_backward.yaml,${PADDLE_SOURCE_DIR}/paddle/phi/ops/yaml/sparse_backward.yaml,${PADDLE_SOURCE_DIR}/paddle/phi/ops/yaml/fused_backward.yaml"
 )
-set(no_sparse_backward_yaml_path
-    "${PADDLE_SOURCE_DIR}/paddle/phi/ops/yaml/backward.yaml,${PADDLE_SOURCE_DIR}/paddle/phi/ops/yaml/inconsistent/dygraph_backward.yaml"
-)
+
 set(tmp_forwards_cc_path
     "${PADDLE_SOURCE_DIR}/paddle/fluid/eager/api/generated/eager_generated/forwards/tmp_dygraph_functions.cc"
 )
@@ -70,7 +68,7 @@ add_custom_target(
   COMMAND
     "${PYTHON_EXECUTABLE}"
     "${PADDLE_SOURCE_DIR}/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py"
-    "--api_yaml_path=${api_yaml_path},${fwd_api_yaml_path},${no_sparse_backward_yaml_path}"
+    "--api_yaml_path=${api_yaml_path},${fwd_api_yaml_path},${backward_yaml_path}"
     "--source_path=${tmp_python_c_source_path}"
     "--header_path=${tmp_python_c_header_path}"
   COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_python_c_source_path}

paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py

@@ -118,7 +118,10 @@ def ReadFwdFile(filepath):
     contents = yaml.load(f, Loader=yaml.FullLoader)
     f.close()
     # not all fused ops support dygraph
-    if filepath.endswith("fused_ops.yaml") is True:
+    if (
+        filepath.endswith("fused_ops.yaml") is True
+        or filepath.endswith("fused_backward.yaml") is True
+    ):
         new_apis = [
             api
             for api in contents

paddle/fluid/eager/auto_code_generator/generator/eager_gen.py

@@ -3357,8 +3357,12 @@ def GenerateForwardHFile(filepath, forward_function_declaration_str):
             generator_grad = DygraphForwardAndNodesGenerator(
                 backward_yaml_paths[i], backward_yaml_paths[i], all_bw, all_bw
             )
-        else:
+        elif backward_yaml_path.endswith('/dygraph_backward.yaml'):
             continue
+        else:
+            generator_grad = DygraphForwardAndNodesGenerator(
+                backward_yaml_path, backward_yaml_path
+            )
 
         generator_grad.run(True)
 

paddle/fluid/pir/dialect/op_generator/ops_api_gen.py

@@ -251,8 +251,6 @@ def __init__(self) -> None:
 
     def _need_skip(self, op_info, op_name):
         if op_name.endswith("_grad"):
-            if op_info.is_sparse_op or op_info.is_fused_op:
-                return True
             if op_name.endswith(("double_grad", "_grad_grad", "triple_grad")):
                 return True
             if op_name[:-5] in NO_NEED_GEN_STATIC_ONLY_APIS:

paddle/phi/api/generator/intermediate_api_gen.py

@@ -15,7 +15,7 @@
 import argparse
 
 import yaml
-from api_gen import ForwardAPI
+from api_gen import ForwardAPI, backward_api_black_list
 from dist_api_gen import DistForwardAPI
 from sparse_api_gen import SparseAPI
 
@@ -131,12 +131,17 @@ def generate_intermediate_api(
 
     dygraph_header_file.write(sparse_namespace_pair[0])
     dygraph_source_file.write(sparse_namespace_pair[0])
-
-    with open(sparse_api_yaml_path, 'r') as f:
-        sparse_apis = yaml.load(f, Loader=yaml.FullLoader)
+    sparse_apis = []
+    for each_sparse_api_yaml in sparse_api_yaml_path:
+        with open(each_sparse_api_yaml, 'r') as f:
+            sparse_api_list = yaml.load(f, Loader=yaml.FullLoader)
+            if sparse_api_list:
+                sparse_apis.extend(sparse_api_list)
 
     for api in sparse_apis:
         sparse_api = SparseAPI(api)
+        if sparse_api.api in backward_api_black_list:
+            continue
         if sparse_api.is_dygraph_api:
             dygraph_header_file.write(sparse_api.gene_api_declaration())
             dygraph_source_file.write(sparse_api.gene_api_code())
@@ -164,6 +169,7 @@ def main():
 
     parser.add_argument(
         '--sparse_api_yaml_path',
+        nargs='+',
         help='path to sparse api yaml file',
         default='paddle/phi/ops/yaml/sparse_ops.yaml',
     )

paddle/phi/api/generator/sparse_api_gen.py

@@ -16,7 +16,7 @@
 
 import yaml
 from api_base import PREFIX_TENSOR_NAME
-from api_gen import ForwardAPI
+from api_gen import ForwardAPI, backward_api_black_list
 
 
 class SparseAPI(ForwardAPI):
@@ -438,11 +438,13 @@ def source_include(header_file_path):
 #include "paddle/phi/infermeta/binary.h"
 #include "paddle/phi/infermeta/ternary.h"
 #include "paddle/phi/infermeta/multiary.h"
+#include "paddle/phi/infermeta/backward.h"
 #include "paddle/utils/none.h"
 
 #include "paddle/phi/infermeta/sparse/unary.h"
 #include "paddle/phi/infermeta/sparse/binary.h"
 #include "paddle/phi/infermeta/sparse/multiary.h"
+#include "paddle/phi/infermeta/sparse/backward.h"
 
 COMMON_DECLARE_int32(low_precision_op_list);
 COMMON_DECLARE_bool(benchmark);
@@ -467,8 +469,13 @@ def api_namespace():
 
 
 def generate_api(api_yaml_path, header_file_path, source_file_path):
-    with open(api_yaml_path, 'r') as f:
-        apis = yaml.load(f, Loader=yaml.FullLoader)
+    apis = []
+
+    for each_api_yaml in api_yaml_path:
+        with open(each_api_yaml, 'r') as f:
+            api_list = yaml.load(f, Loader=yaml.FullLoader)
+            if api_list:
+                apis.extend(api_list)
     header_file = open(header_file_path, 'w')
     source_file = open(source_file_path, 'w')
 
@@ -483,7 +490,10 @@ def generate_api(api_yaml_path, header_file_path, source_file_path):
     source_file.write(namespace[0])
 
     for api in apis:
+
         sparse_api = SparseAPI(api)
+        if sparse_api.api in backward_api_black_list:
+            continue
         if sparse_api.is_dygraph_api:
             sparse_api.is_dygraph_api = False
         header_file.write(sparse_api.gene_api_declaration())
@@ -503,6 +513,7 @@ def main():
     parser.add_argument(
         '--api_yaml_path',
         help='path to sparse api yaml file',
+        nargs='+',
         default='paddle/phi/ops/yaml/sparse_ops.yaml',
     )
 

paddle/phi/api/lib/CMakeLists.txt

@@ -202,8 +202,9 @@ endif()
 execute_process(
   COMMAND
     ${PYTHON_EXECUTABLE} ${sparse_api_gen_file} --api_yaml_path
-    ${sparse_api_yaml_file} --api_header_path ${sparse_api_header_file_tmp}
-    --api_source_path ${sparse_api_source_file_tmp})
+    ${sparse_api_yaml_file} ${sparse_bw_api_yaml_file} --api_header_path
+    ${sparse_api_header_file_tmp} --api_source_path
+    ${sparse_api_source_file_tmp})
 
 # generate backward sparse api
 execute_process(
@@ -226,17 +227,18 @@ if(WITH_DISTRIBUTE)
     COMMAND
       ${PYTHON_EXECUTABLE} ${im_api_gen_file} --api_yaml_path ${api_yaml_file}
       ${bw_api_yaml_file} ${legacy_api_yaml_file} ${legacy_bw_api_yaml_file}
-      --sparse_api_yaml_path ${sparse_api_yaml_file} --dygraph_api_header_path
-      ${dygraph_api_header_file_tmp} --dygraph_api_source_path
-      ${dygraph_api_source_file_tmp} --gen_dist_branch)
+      --sparse_api_yaml_path ${sparse_api_yaml_file} ${sparse_bw_api_yaml_file}
+      --dygraph_api_header_path ${dygraph_api_header_file_tmp}
+      --dygraph_api_source_path ${dygraph_api_source_file_tmp}
+      --gen_dist_branch)
 else()
   execute_process(
     COMMAND
       ${PYTHON_EXECUTABLE} ${im_api_gen_file} --api_yaml_path ${api_yaml_file}
       ${bw_api_yaml_file} ${legacy_api_yaml_file} ${legacy_bw_api_yaml_file}
-      --sparse_api_yaml_path ${sparse_api_yaml_file} --dygraph_api_header_path
-      ${dygraph_api_header_file_tmp} --dygraph_api_source_path
-      ${dygraph_api_source_file_tmp})
+      --sparse_api_yaml_path ${sparse_api_yaml_file} ${sparse_bw_api_yaml_file}
+      --dygraph_api_header_path ${dygraph_api_header_file_tmp}
+      --dygraph_api_source_path ${dygraph_api_source_file_tmp})
 endif()
 
 # generate tensor and tensor operants file