[AutoParallel] Support intermediate_api baichuan test (#9988)

Author: Function-Samuel

tests/test_tipc/static/auto_parallel/baichuan2/N4C32/intermediate_api_baichuan-inc-baichuan-2-13b_pretrain_dy2st_bs32_bf16_DP1_MP4_PP1_Sharding8_Stage1.sh

@@ -0,0 +1,26 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+param="model_item=intermediate_api_baichuan-inc-baichuan-2-13b_pretrain_dy2st "
+param+="run_mode=DP1_MP4_PP1_Sharding8_Stage1 "
+param+="device_num=N4C32 "
+param+="global_batch_size=32 "
+param+="nnodes=4 "
+param+="model_type=baichuan2_13b "
+param+="intermediate_api=intermediate_api_ "
+
+cd ./tests
+bash ./test_tipc/static/auto_parallel/baichuan2/benchmark_common/prepare.sh
+
+bash -c "${param} bash ./test_tipc/static/auto_parallel/baichuan2/benchmark_common/run_benchmark.sh"

tests/test_tipc/static/auto_parallel/baichuan2/benchmark_common/run_benchmark.sh

@@ -24,6 +24,9 @@ function _set_params(){
     fp_item="bf16"
     MODEL_TYPE=${model_type:-"baichuan2_13b"}
 
+    # for intermediate api
+    intermediate_api=${intermediate_api:-""}
+
     ip_lists=($(echo $TRAINER_INSTANCES | tr ',' ' '))
     master_ip=${ip_lists[0]}
     nnodes=${nnodes:-1}
@@ -170,17 +173,17 @@ function _train(){
         train_cmd="python -u -m paddle.distributed.launch --gpus=0,1,2,3,4,5,6,7 \
             --nnodes 1 --nproc_per_node 8 \
             --log_dir mylog run_pretrain_auto.py \
-            ./pretrain_config_${MODEL_TYPE}/pretrain-${MODEL_TYPE}.json"
+            ./pretrain_config_${MODEL_TYPE}/${intermediate_api}pretrain-${MODEL_TYPE}.json"
         ;;
     N4C32) echo "Run with: device_num=${device_num} run_mode=${run_mode}"
         train_cmd="python -u -m paddle.distributed.launch --gpus=0,1,2,3,4,5,6,7 \
             --log_dir mylog run_pretrain_auto.py \
-            ./pretrain_config_${MODEL_TYPE}/pretrain-${MODEL_TYPE}.json"
+            ./pretrain_config_${MODEL_TYPE}/${intermediate_api}pretrain-${MODEL_TYPE}.json"
         ;;
     *) echo "Run with: device_num=${device_num}, run_mode=${run_mode}"
         train_cmd="python -u -m paddle.distributed.launch --gpus=0,1,2,3,4,5,6,7 \
             --log_dir mylog run_pretrain_auto.py \
-            ./pretrain_config_${MODEL_TYPE}/pretrain-${MODEL_TYPE}.json"
+            ./pretrain_config_${MODEL_TYPE}/${intermediate_api}pretrain-${MODEL_TYPE}.json"
         ;;
     esac
     cd ../llm/auto_parallel/llama

tests/test_tipc/static/auto_parallel/baichuan2/pretrain_config_baichuan2_13b/intermediate_api_pretrain-baichuan2_13b.json

@@ -0,0 +1,57 @@
+{
+    "model_name_or_path": "baichuan-inc/Baichuan2-13B-Base",
+    "tokenizer_name_or_path": "baichuan-inc/Baichuan2-13B-Base",
+    "input_dir": "./data",
+    "output_dir": "./checkpoints/baichuan2_13b_ckpts",
+    "split": "949,50,1",
+    "to_static": true,
+    "pipeline_parallel_degree": 1,
+    "tensor_parallel_degree": 4,
+    "virtual_pp_degree": 1,
+    "weight_decay": 0.01,
+    "warmup_ratio": 0.01,
+    "max_grad_norm": 1.0,
+    "learning_rate": 0.00003,
+    "min_learning_rate": 0.000003,
+    "max_steps": 200,
+    "logging_steps": 5,
+    "eval_steps": 10000,
+    "save_steps": 1000,
+    "continue_training": 0,
+    "do_train": true,
+    "do_eval": false,
+    "do_predict": false,
+    "disable_tqdm": true,
+    "save_total_limit": 2,
+    "device": "gpu",
+    "dataloader_num_workers": 1,
+    "distributed_dataloader": 0,
+    "enable_auto_parallel": 1,
+    "per_device_train_batch_size": 2,
+    "gradient_accumulation_steps": 2,
+    "per_device_eval_batch_size": 1,
+    "recompute": false,
+    "recompute_use_reentrant": true,
+    "recompute_granularity": "full",
+    "pp_recompute_interval": 0,
+    "bf16": true,
+    "fp16_opt_level": "O2",
+    "amp_master_grad": true,
+    "fuse_attention_ffn": true,
+    "fuse_attention_qkv": true,
+    "use_flash_attention": true,
+    "fused_linear": 1,
+    "fused_linear_param_grad_add": 1,
+    "use_fused_rope": true,
+    "use_fused_rms_norm": true,
+    "max_seq_length": 4096,
+    "sequence_parallel": 1,
+    "sharding": "stage1",
+    "sharding_parallel_degree": 8,
+    "sharding_parallel_config": "enable_tensor_fusion enable_overlap",
+    "tensor_parallel_config": "enable_mp_async_allreduce replace_with_parallel_cross_entropy",
+    "data_parallel_config": "enable_allreduce_avg_in_gradinent_scale gradient_sync_after_accumulate",
+    "pipeline_parallel_config": "enable_send_recv_overlap enable_split_backward",
+    "model_type": "llama_network",
+    "use_intermediate_api": true
+}