add unittests

Author: haohongxiang

python/paddle/distributed/fleet/meta_optimizers/dygraph_optimizer/hybrid_parallel_optimizer.py

@@ -148,15 +148,15 @@ def _dygraph_clip(self, params_grads):
             x=max_global_norm,
             y=layers.elementwise_max(
                 x=global_norm_var_fp32, y=max_global_norm))
+        clip_var_fp16 = paddle.cast(clip_var, paddle.float16)
         for p, g in params_grads:
             if g is None:
                 continue
             if getattr(p, 'need_clip', True) is False:
                 params_and_grads.append((p, g))
                 continue
             if p.dtype == paddle.float16:
-                new_grad = layers.elementwise_mul(
-                    x=g, y=paddle.cast(clip_var, paddle.float16))
+                new_grad = layers.elementwise_mul(x=g, y=clip_var_fp16)
             else:
                 new_grad = layers.elementwise_mul(x=g, y=clip_var)
             params_and_grads.append((p, new_grad))

python/paddle/fluid/tests/unittests/hybrid_parallel_pp_amp.py

@@ -61,11 +61,14 @@ def test_pp_model(self):
         rank_id = dist.get_rank()
         set_random_seed(1024, dp_id, rank_id)
 
+        grad_clip = paddle.nn.ClipGradByGlobalNorm(1.0)
+
         #construct model a
         model_a = AlexNet(10)
         scheduler_a = paddle.optimizer.lr.PiecewiseDecay(
             boundaries=[2], values=[0.001, 0.002], verbose=True)
         optimizer_a = paddle.optimizer.SGD(learning_rate=scheduler_a,
+                                           grad_clip=grad_clip,
                                            parameters=model_a.parameters())
 
         scaler_a = paddle.amp.GradScaler(init_loss_scaling=2**5)
@@ -80,6 +83,7 @@ def test_pp_model(self):
         scheduler_b = paddle.optimizer.lr.PiecewiseDecay(
             boundaries=[2], values=[0.001, 0.002], verbose=True)
         optimizer_b = paddle.optimizer.SGD(learning_rate=scheduler_b,
+                                           grad_clip=grad_clip,
                                            parameters=model_b.parameters())
         model_b = fleet.distributed_model(model_b)
         optimizer_b = fleet.distributed_optimizer(optimizer_b)

python/paddle/fluid/tests/unittests/hybrid_parallel_pp_fp16.py

@@ -61,11 +61,14 @@ def test_pp_model(self):
         rank_id = dist.get_rank()
         set_random_seed(1024, dp_id, rank_id)
 
+        grad_clip = paddle.nn.ClipGradByGlobalNorm(1.0)
+
         #construct model a
         model_a = AlexNet(10)
         scheduler_a = paddle.optimizer.lr.PiecewiseDecay(
             boundaries=[2], values=[0.001, 0.002], verbose=True)
         optimizer_a = paddle.optimizer.SGD(learning_rate=scheduler_a,
+                                           grad_clip=grad_clip,
                                            parameters=model_a.parameters())
 
         scaler_a = paddle.amp.GradScaler(init_loss_scaling=2**5)
@@ -75,6 +78,7 @@ def test_pp_model(self):
         scheduler_b = paddle.optimizer.lr.PiecewiseDecay(
             boundaries=[2], values=[0.001, 0.002], verbose=True)
         optimizer_b = paddle.optimizer.SGD(learning_rate=scheduler_b,
+                                           grad_clip=grad_clip,
                                            parameters=model_b.parameters())
 
         param_len = len(model_a.parameters())