Merge branch 'PaddlePaddle:develop' into develop

Author: Thinksky5124

data/k400/train_small_frames.list

@@ -40,12 +40,14 @@ def train_model(cfg,
 
     Args:
         cfg (dict): configuration.
-        weights (str): weights path for finetuning.
-        parallel (bool): Whether multi-cards training. Default: True.
-        validate (bool): Whether to do evaluation. Default: False.
-        amp (bool): Whether to use automatic mixed precision during training. Default: False.
-        use_fleet (bool):
-        profiler_options (str): Activate the profiler function Default: None.
+        weights (str, optional): weights path for finetuning. Defaults to None.
+        parallel (bool, optional): whether multi-cards training. Defaults to True.
+        validate (bool, optional): whether to do evaluation. Defaults to True.
+        amp (bool, optional): whether to use automatic mixed precision during training. Defaults to False.
+        max_iters (int, optional): max running iters in an epoch. Defaults to None.
+        use_fleet (bool, optional): whether to use fleet. Defaults to False.
+        profiler_options (str, optional): configuration for the profiler function. Defaults to None.
+
     """
     if use_fleet:
         fleet.init(is_collective=True)
@@ -193,8 +195,9 @@ def train_model(cfg,
                         scaler.minimize(optimizer, scaled)
                         optimizer.clear_grad()
                 else:  # general case
-                    # 4.2 backward
+                    # Loss scaling
                     scaled = scaler.scale(avg_loss)
+                    # 4.2 backward
                     scaled.backward()
                     # 4.3 minimize
                     scaler.minimize(optimizer, scaled)
@@ -354,10 +357,10 @@ def evaluate(best):
             save(
                 optimizer.state_dict(),
                 osp.join(output_dir,
-                         model_name + f"_epoch_{epoch+1:05d}.pdopt"))
+                         model_name + f"_epoch_{epoch + 1:05d}.pdopt"))
             save(
                 model.state_dict(),
                 osp.join(output_dir,
-                         model_name + f"_epoch_{epoch+1:05d}.pdparams"))
+                         model_name + f"_epoch_{epoch + 1:05d}.pdparams"))
 
     logger.info(f'training {model_name} finished')

data/k400/train_small_videos.list

@@ -134,8 +134,8 @@ func_sed_params "$FILENAME" "${line_export_py_2}" "null"
 func_sed_params "$FILENAME" "${line_export_py_3}" "null"
 func_sed_params "$FILENAME" "${line_python}"  "$python"
 
-# 末尾加上--log_interval=1，以便输出足量数据
-set_log_interval_cmd="sed -i '${line_norm_train}s/.*/& -o log_interval=1/' '${filename}'"
+# 末尾加上--max_iters=30和--log_interval=1，以便运行并输出足量数据
+set_log_interval_cmd="sed -i '${line_norm_train}s/.*/& --max_iters=30 -o log_interval=1/' '${filename}'"
 eval $set_log_interval_cmd
 
 # 去掉--validate，benchmark不需要validate

paddlevideo/tasks/train.py

@@ -13,7 +13,7 @@ train_infer_video_dir:null
 null:null
 ##
 trainer:norm_train
-norm_train:main.py -c configs/recognition/agcn/agcn_fsd.yaml --seed 1234 --max_iters=10 -o DATASET.train.file_path="data/fsd10/FSD_train_data.npy" -o DATASET.train.label_path="data/fsd10/FSD_train_label.npy" -o DATASET.test.file_path="data/fsd10/FSD_train_data.npy"
+norm_train:main.py -c configs/recognition/agcn/agcn_fsd.yaml --seed 1234 -o DATASET.train.file_path="data/fsd10/FSD_train_data.npy" -o DATASET.train.label_path="data/fsd10/FSD_train_label.npy" -o DATASET.test.file_path="data/fsd10/FSD_train_data.npy"
 pact_train:null
 fpgm_train:null
 distill_train:null

test_tipc/benchmark_train.sh

@@ -13,7 +13,7 @@ train_model_name:null
 -o DATASET.train.file_path:null
 ##
 trainer:norm_train
-norm_train:main.py --validate -c configs/localization/bmn.yaml --seed 1234 --max_iters=30
+norm_train:main.py --validate -c configs/localization/bmn.yaml --seed 1234
 pact_train:null
 fpgm_train:null
 distill_train:null

test_tipc/configs/AGCN/AGCN_train_infer_python.txt

@@ -13,7 +13,7 @@ train_infer_video_dir:null
 null:null
 ##
 trainer:norm_train
-norm_train:main.py -c configs/recognition/stgcn/stgcn_fsd.yaml --seed 1234 --max_iters=10 -o DATASET.train.file_path="data/fsd10/FSD_train_data.npy" -o DATASET.train.label_path="data/fsd10/FSD_train_label.npy" -o DATASET.test.file_path="data/fsd10/FSD_train_data.npy"
+norm_train:main.py -c configs/recognition/stgcn/stgcn_fsd.yaml --seed 1234 -o DATASET.train.file_path="data/fsd10/FSD_train_data.npy" -o DATASET.train.label_path="data/fsd10/FSD_train_label.npy" -o DATASET.test.file_path="data/fsd10/FSD_train_data.npy"
 pact_train:null
 fpgm_train:null
 distill_train:null

test_tipc/configs/BMN/BMN_train_infer_python.txt

@@ -13,7 +13,7 @@ train_infer_video_dir:null
 -o DATASET.train.file_path:'data/k400/train_small_videos.list' -o DATASET.valid.file_path='data/k400/train_small_videos.list' -o DATASET.test.file_path='data/k400/train_small_videos.list'
 ##
 trainer:norm_train
-norm_train:main.py --validate -c configs/recognition/slowfast/slowfast.yaml --seed 1234 --max_iters=30
+norm_train:main.py --validate -c configs/recognition/slowfast/slowfast.yaml --seed 1234
 pact_train:null
 fpgm_train:null
 distill_train:null

test_tipc/configs/STGCN/STGCN_train_infer_python.txt

@@ -112,7 +112,7 @@ Linux端基础训练预测功能测试的主程序为`test_train_inference_pytho
 python3.7 test_tipc/compare_results.py --gt_file="test_tipc/results/python_*.txt"  --log_file="test_tipc/output/python_*.log" --atol=1e-3 --rtol=1e-3
 ```
 
-参数介绍：  
+参数介绍：
 - gt_file： 指向事先保存好的预测结果路径，支持*.txt 结尾，会自动索引*.txt格式的文件，文件默认保存在test_tipc/result/ 文件夹下
 - log_file: 指向运行test_tipc/test_train_inference_python.sh 脚本的infer模式保存的预测日志，预测日志中打印的有预测结果，比如：预测文本，类别等等，同样支持python_infer_*.log格式传入
 - atol: 设置的绝对误差

test_tipc/configs/SlowFast/SlowFast_train_infer_python.txt

@@ -145,16 +145,16 @@ function func_inference(){
     for use_gpu in ${use_gpu_list[*]}; do
         if [ ${use_gpu} = "False" ] || [ ${use_gpu} = "cpu" ]; then
             for use_mkldnn in ${use_mkldnn_list[*]}; do
-                if [ ${use_mkldnn} = "False" ] && [ ${_flag_quant} = "True" ]; then
+                if [[ ${use_mkldnn} = "False" ]] && [[ ${_flag_quant} = "True" ]]; then
                     continue
                 fi
                 for threads in ${cpu_threads_list[*]}; do
                     for batch_size in ${batch_size_list[*]}; do
                         for precision in ${precision_list[*]}; do
-                            if [ ${use_mkldnn} = "False" ] && [ ${precision} = "fp16" ]; then
+                            if [[ ${use_mkldnn} = "False" ]] && [[ ${precision} = "fp16" ]]; then
                                 continue
                             fi # skip when enable fp16 but disable mkldnn
-                            if [ ${_flag_quant} = "True" ] && [ ${precision} != "int8" ]; then
+                            if [[ ${_flag_quant} = "True" ]] && [[ ${precision} != "int8" ]]; then
                                 continue
                             fi # skip when quant model inference but precision is not int8
                             set_precision=$(func_set_params "${precision_key}" "${precision}")
@@ -185,10 +185,10 @@ function func_inference(){
                     if [[ ${_flag_quant} = "False" ]] && [[ ${precision} =~ "int8" ]]; then
                         continue
                     fi
-                    if [[ ${precision} =~ "fp16" || ${precision} =~ "int8" ]] && [ ${use_trt} = "False" ]; then
+                    if [[ ${precision} =~ "fp16" || ${precision} =~ "int8" ]] && [[ ${use_trt} = "False" ]]; then
                         continue
                     fi
-                    if [[ ${use_trt} = "False" || ${precision} =~ "int8" ]] && [ ${_flag_quant} = "True" ]; then
+                    if [[ ${use_trt} = "False" || ${precision} =~ "int8" ]] && [[ ${_flag_quant} = "True" ]]; then
                         continue
                     fi
                     for batch_size in ${batch_size_list[*]}; do
@@ -309,6 +309,10 @@ else
                     run_export=${export_value2}
                 else
                     run_train=${norm_trainer}
+                    if [[ ${MODE} != "benchmark_train" ]] && [[ ! ${MODE} =~ "whole_train" ]]; then
+                        # 训练参数末尾加上--max_iters=30和--log_interval=1，以便运行并输出足量数据
+                        run_train=${run_train}" --max_iters=30"
+                    fi
                     run_export=${norm_export}
                 fi