Merge pull request #603 from ForFishes/add_yaml

Add yaml config for gpt model

Author: fuyinno4

examples/gpt/3D_parallelism/configs.yaml

@@ -0,0 +1,61 @@
+# 175 B
+PreTraining:
+  device: gpu
+  max_steps: 500000
+  num_train_epochs: 1
+  seed: 1024
+  use_recompute: True
+  batch_size:
+    global_batch_size: 8
+    local_batch_size: 
+    micro_batch_size: 1
+  mix_precision:
+    use_pure_fp16: True
+    scale_loss: 32768.0
+  logging_freq: 1
+  eval_freq: 500
+  eval_iters: 10
+  dataset:
+    input_dir: ./data
+    split: '949,50,1'
+    max_seq_len: 2048  # 
+  save_load:
+    save_steps: 1000
+    output_dir: ./output
+    ckpt_dir: 
+
+  Model:
+    vocab_size: 51200
+    hidden_size: 12288
+    num_layers: 96
+    num_attention_heads: 96
+    ffn_hidden_size: 
+    hidden_dropout_prob: 0.1
+    attention_probs_dropout_prob: 0.1
+    max_position_embeddings: 1024
+    type_vocab_size: 16
+    initializer_range: 0.02
+
+  Distributed:
+    dp_degree: 1
+    mp_degree: 8
+    pp_degree: 16
+    sharding:
+      sharding_degree: 1
+      sharding_stage: 1
+      sharding_offload: False
+
+  Optimizer:
+    # name: Adam
+    weight_decay: 0.01
+    adam_beta1: 0.9
+    adam_beta2: 0.999
+    adam_epsilon: 1.0e-8
+    lr:
+      # name: consine
+      decay_steps: 360000
+      # max_steps: 500000
+      warmup_rate: 0.01
+      max_lr: 1.0e-5
+      min_lr: 5.0e-5
+    grad_clip: 1.0

examples/gpt/3D_parallelism/run.sh

@@ -16,29 +16,8 @@ export PYTHONPATH=$PYTHONPATH:../../../
 
 log_dir=dp2_pp2_mp2
 rm -rf $log_dir
+export FLAGS_enable_eager_mode=0
 
-# 345M
+# 175B
 python -m paddle.distributed.launch --log_dir $log_dir --devices "0,1,2,3,4,5,6,7" run_pretrain.py \
-    --input_dir "./data"\
-    --output_dir "output"\
-    --vocab_size 50304\
-    --hidden_size 1024\
-    --num_layers 24\
-    --num_attention_heads 16\
-    --max_seq_len 1024\
-    --weight_decay 0.01\
-    --grad_clip 1.0\
-    --max_steps 500000\
-    --save_steps 100000\
-    --decay_steps 320000\
-    --device gpu\
-    --eval_freq 1000\
-    --warmup_rate 0.01\
-    --scale_loss 32768\
-    --global_batch_size 32\
-    --micro_batch_size 1\
-    --dp_degree 1\
-    --mp_degree 8\
-    --pp_degree 1\
-    --use_recompute True\
-    --use_pure_fp16 False
+    -c ./configs.yaml

examples/gpt/3D_parallelism/run_pretrain.py

@@ -19,7 +19,7 @@
 import time
 import sys
 sys.path.append("..")
-from args import parse_args
+from tools import parse_args, parse_yaml
 
 import numpy as np
 import paddle
@@ -295,5 +295,5 @@ def do_train(args):
 
 
 if __name__ == "__main__":
-    args = parse_args()
+    args = parse_yaml(parse_args().config)
     do_train(args)