[PRETRAIN] Change some hyper parameters of ernie-1.0 (#1344)

* fix ernie, Normal->TruncatedNormal,  smaller steps. 

* fix by set attn_mask to -1e4

Author: ZHUI

examples/language_model/data_tools/ernie_dataset.py

@@ -203,6 +203,8 @@ def pad_and_convert_to_numpy(tokens, tokentypes, masked_positions,
     # Padding mask.
     padding_mask_np = np.array(
         [1] * num_tokens + [0] * padding_length, dtype=np.float32)
+    padding_mask_np = (1 - padding_mask_np) * -1e4
+
     padding_mask_np = padding_mask_np.reshape([1, 1, -1])
     # Lables and loss mask.
     labels = [-1] * max_seq_length

examples/language_model/ernie-1.0/run_gb512_s400.sh renamed to examples/language_model/ernie-1.0/run_gb512_s200.sh

@@ -30,12 +30,12 @@ PYTHONPATH=../../../  python -u  -m paddle.distributed.launch \
     --use_recompute false \
     --max_lr 0.0001 \
     --min_lr 0.00001 \
-    --max_steps 4000000 \
+    --max_steps 2000000 \
     --save_steps 50000 \
     --checkpoint_steps 5000 \
-    --decay_steps 3960000 \
+    --decay_steps 1980000 \
     --weight_decay 0.01 \
-    --warmup_rate 0.0025 \
+    --warmup_rate 0.005 \
     --grad_clip 1.0 \
     --logging_freq 20\
     --num_workers 2 \

paddlenlp/transformers/ernie/modeling.py

@@ -293,8 +293,9 @@ def __init__(self,
         super(ErnieModel, self).__init__()
         self.pad_token_id = pad_token_id
         self.initializer_range = initializer_range
-        weight_attr = paddle.ParamAttr(initializer=nn.initializer.Normal(
-            mean=0.0, std=self.initializer_range))
+        weight_attr = paddle.ParamAttr(
+            initializer=nn.initializer.TruncatedNormal(
+                mean=0.0, std=self.initializer_range))
         self.embeddings = ErnieEmbeddings(
             vocab_size, hidden_size, hidden_dropout_prob,
             max_position_embeddings, type_vocab_size, pad_token_id, weight_attr)
@@ -683,8 +684,9 @@ class ErnieForPretraining(ErniePretrainedModel):
     def __init__(self, ernie):
         super(ErnieForPretraining, self).__init__()
         self.ernie = ernie
-        weight_attr = paddle.ParamAttr(initializer=nn.initializer.Normal(
-            mean=0.0, std=self.ernie.initializer_range))
+        weight_attr = paddle.ParamAttr(
+            initializer=nn.initializer.TruncatedNormal(
+                mean=0.0, std=self.ernie.initializer_range))
         self.cls = ErniePretrainingHeads(
             self.ernie.config["hidden_size"],
             self.ernie.config["vocab_size"],