refine optimizer create accumulators (#50188)

* refine optimizer create accumulators

* refine

Author: wanghuancoder

python/paddle/optimizer/adadelta.py

@@ -145,8 +145,11 @@ def _create_accumulators(self, block, parameters):
             parameters = parameters.get('params')
 
         for p in parameters:
+            if p.name in self._already_create_accumulater:
+                continue
             self._add_accumulator(self._avg_squared_grad_acc_str, p)
             self._add_accumulator(self._avg_squared_update_acc_str, p)
+            self._already_create_accumulater.add(p.name)
 
     def _append_optimize_op(self, block, param_and_grad):
         if isinstance(param_and_grad, dict):

python/paddle/optimizer/adagrad.py

@@ -139,11 +139,14 @@ def _create_accumulators(self, block, parameters):
             parameters = self._update_param_group(parameters)
 
         for p in parameters:
+            if p.name in self._already_create_accumulater:
+                continue
             self._add_accumulator(
                 self._moment_acc_str,
                 p,
                 fill_value=self.initial_accumulator_value,
             )
+            self._already_create_accumulater.add(p.name)
 
     def _append_optimize_op(self, block, param_and_grad):
         assert isinstance(block, framework.Block)

python/paddle/optimizer/adam.py

@@ -317,9 +317,12 @@ def _create_accumulators(self, block, parameters):
 
         # Create accumulator tensors for first and second moments
         for p in parameters:
+            if p.name in self._already_create_accumulater:
+                continue
             if self._multi_precision and self._is_dtype_fp16_or_bf16(p.dtype):
                 master_p = self._create_master_weight(p)
                 self._add_moments_pows(master_p)
+                self._already_create_accumulater.add(p.name)
                 continue
             if (
                 self._is_dtype_fp16_or_bf16(p.dtype)
@@ -330,6 +333,7 @@ def _create_accumulators(self, block, parameters):
                     "Consider using multi_precision=True option of the Adam optimizer."
                 )
             self._add_moments_pows(p)
+            self._already_create_accumulater.add(p.name)
 
     def _append_optimize_op(self, block, param_and_grad):
         assert isinstance(block, framework.Block)

python/paddle/optimizer/adamax.py

@@ -176,6 +176,8 @@ def _create_accumulators(self, block, parameters):
 
         # Create accumulator tensors for first moment and infinity norm
         for p in parameters:
+            if p.name in self._already_create_accumulater:
+                continue
             self._add_accumulator(self._moment_acc_str, p)
             self._add_accumulator(self._inf_norm_acc_str, p)
             self._add_accumulator(
@@ -184,6 +186,7 @@ def _create_accumulators(self, block, parameters):
                 fill_value=self._beta1,
                 shape=[1],
             )
+            self._already_create_accumulater.add(p.name)
 
     def _append_optimize_op(self, block, param_and_grad):
         assert isinstance(block, framework.Block)

python/paddle/optimizer/adamw.py

@@ -281,6 +281,7 @@ def __init__(
         self._use_multi_tensor = None
         self.regularization = None
         self._auxiliary_vars = {}
+        self._already_create_accumulater = set()
 
     def _set_auxiliary_var(self, key, val):
         self._auxiliary_vars[key] = val
@@ -422,9 +423,12 @@ def _create_accumulators(self, block, parameters):
 
         # Create accumulator tensors for first and second moments
         for p in parameters:
+            if p.name in self._already_create_accumulater:
+                continue
             if self._multi_precision and self._is_dtype_fp16_or_bf16(p.dtype):
                 master_p = self._create_master_weight(p)
                 self._add_moments_pows(master_p)
+                self._already_create_accumulater.add(p.name)
                 continue
             if (
                 self._is_dtype_fp16_or_bf16(p.dtype)
@@ -435,6 +439,7 @@ def _create_accumulators(self, block, parameters):
                     "Consider using multi_precision=True option of the Adam optimizer."
                 )
             self._add_moments_pows(p)
+            self._already_create_accumulater.add(p.name)
 
     def _append_optimize_op(self, block, param_and_grad):
         assert isinstance(block, framework.Block)

python/paddle/optimizer/lamb.py

@@ -190,11 +190,15 @@ def _create_accumulators(self, block, parameters):
 
         # Create accumulator tensors for first and second moments
         for p in parameters:
+            if p.name in self._already_create_accumulater:
+                continue
             if self._multi_precision and p.dtype == core.VarDesc.VarType.FP16:
                 master_p = self._create_master_weight(p)
                 self._add_moments_pows(master_p)
+                self._already_create_accumulater.add(p.name)
             else:
                 self._add_moments_pows(p)
+                self._already_create_accumulater.add(p.name)
 
     def _get_accumulator(self, name, param):
         """Utility function to fetch an accumulator for a parameter

python/paddle/optimizer/momentum.py

@@ -270,9 +270,12 @@ def _create_accumulators(self, block, parameters):
             parameters = self._update_param_group(parameters)
 
         for p in parameters:
+            if p.name in self._already_create_accumulater:
+                continue
             if self._multi_precision and p.dtype == core.VarDesc.VarType.FP16:
                 master_p = self._create_master_weight(p)
                 self._add_accumulator(self._velocity_acc_str, master_p)
+                self._already_create_accumulater.add(p.name)
                 continue
             if (
                 p.dtype == core.VarDesc.VarType.FP16
@@ -283,6 +286,7 @@ def _create_accumulators(self, block, parameters):
                     "Consider using multi_precision=True option of the Momentum optimizer."
                 )
             self._add_accumulator(self._velocity_acc_str, p)
+            self._already_create_accumulater.add(p.name)
 
     def _create_regularization_of_grad(self, param, grad, regularization=None):
         """Create and add backward regularization Operators

python/paddle/optimizer/optimizer.py

@@ -275,6 +275,7 @@ def __init__(
 
         self._param_dict = self._create_multi_tensor_dict()
         self._auxiliary_vars = {}
+        self._already_create_accumulater = set()
 
     def _set_auxiliary_var(self, key, val):
         self._auxiliary_vars[key] = val

python/paddle/optimizer/rmsprop.py

@@ -199,9 +199,12 @@ def _create_accumulators(self, block, parameters):
             parameters = parameters.get('params')
 
         for p in parameters:
+            if p.name in self._already_create_accumulater:
+                continue
             self._add_accumulator(self._momentum_acc_str, p)
             self._add_accumulator(self._mean_square_acc_str, p)
             self._add_accumulator(self._mean_grad_acc_str, p)
+            self._already_create_accumulater.add(p.name)
 
     def _append_optimize_op(self, block, param_and_grad):
         if not isinstance(block, framework.Block):

python/paddle/optimizer/sgd.py

@@ -129,8 +129,11 @@ def _create_accumulators(self, block, parameters):
 
         # Create accumulator tensors for first and second moments
         for p in parameters:
+            if p.name in self._already_create_accumulater:
+                continue
             if self._multi_precision and p.dtype == core.VarDesc.VarType.FP16:
                 master_p = self._create_master_weight(p)
+                self._already_create_accumulater.add(p.name)
                 continue
             if (
                 p.dtype == core.VarDesc.VarType.FP16