From ee581490cf8e6408b9bb8cab053ab6b3f7a794c2 Mon Sep 17 00:00:00 2001 From: hysunflower Date: Mon, 10 Jan 2022 03:01:20 +0000 Subject: [PATCH 1/5] update_gpt, tets=document_fix --- PaddleNLP | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PaddleNLP b/PaddleNLP index b15382b9b7..719485c674 160000 --- a/PaddleNLP +++ b/PaddleNLP @@ -1 +1 @@ -Subproject commit b15382b9b7bfd6c5043930ca6667ad84e8f49653 +Subproject commit 719485c674ca94dc142bbaa23b2c7777ca152f4b From 8109e98c8df37b1953634ea3648dc17398254bac Mon Sep 17 00:00:00 2001 From: hysunflower Date: Mon, 10 Jan 2022 03:22:17 +0000 Subject: [PATCH 2/5] update_gpt, tets=document_fix --- scripts/dynamic_graph_models.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/dynamic_graph_models.sh b/scripts/dynamic_graph_models.sh index d8fde7bb2e..62b18b369d 100644 --- a/scripts/dynamic_graph_models.sh +++ b/scripts/dynamic_graph_models.sh @@ -74,7 +74,8 @@ dy_video_TimeSformer(){ cd ${cur_model_path}/benchmark/TimeSformer/ pip install scikit-image==0.18.2 pip install pooch==1.5.2 - bash run_all.sh local + pip install lmdb + bash run_all.sh local rm -rf ${BENCHMARK_ROOT}/PaddleVideo/ # 避免数据集占用docker内过多空间,在执行最后一个模型后删掉 } @@ -256,7 +257,7 @@ dy_tsn(){ cur_model_path=${BENCHMARK_ROOT}/PaddleVideo cd ${cur_model_path} - pip install wget av + pip install wget av lmdb pip install scikit-image==0.18.2 pip install pooch==1.5.2 # Prepare pretrained modles @@ -347,7 +348,7 @@ dy_slowfast(){ cd ${cur_model_path} pip install tqdm pip install decord - pip install pandas av + pip install pandas av lmdb pip install scikit-image==0.18.2 pip install pooch==1.5.2 # Prepare data @@ -461,7 +462,7 @@ dy_tsm(){ cur_model_path=${BENCHMARK_ROOT}/PaddleVideo cd ${cur_model_path} - pip install wget av decord + pip install wget av decord lmdb pip install scikit-image==0.18.2 pip install pooch==1.5.2 # Prepare pretrained modles From 853a76c7f7bf9c9dfbd326c37e9806f6ab6314f6 Mon Sep 17 00:00:00 2001 From: hysunflower Date: Mon, 10 Jan 2022 03:30:25 +0000 Subject: [PATCH 3/5] update_gpt, tests=document_fix --- scripts/dynamic_graph_models.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/dynamic_graph_models.sh b/scripts/dynamic_graph_models.sh index 62b18b369d..5e98490a9d 100644 --- a/scripts/dynamic_graph_models.sh +++ b/scripts/dynamic_graph_models.sh @@ -72,9 +72,9 @@ dy_video_TimeSformer(){ echo "dy_video_TimeSformer" cur_model_path=${BENCHMARK_ROOT}/PaddleVideo/ cd ${cur_model_path}/benchmark/TimeSformer/ + pip install -r requirements.txt pip install scikit-image==0.18.2 pip install pooch==1.5.2 - pip install lmdb bash run_all.sh local rm -rf ${BENCHMARK_ROOT}/PaddleVideo/ # 避免数据集占用docker内过多空间,在执行最后一个模型后删掉 } @@ -257,7 +257,8 @@ dy_tsn(){ cur_model_path=${BENCHMARK_ROOT}/PaddleVideo cd ${cur_model_path} - pip install wget av lmdb + pip install -r requirements.txt + pip install wget av pip install scikit-image==0.18.2 pip install pooch==1.5.2 # Prepare pretrained modles @@ -346,9 +347,10 @@ dy_seg(){ dy_slowfast(){ cur_model_path=${BENCHMARK_ROOT}/PaddleVideo cd ${cur_model_path} + pip install -r requirements.txt pip install tqdm pip install decord - pip install pandas av lmdb + pip install pandas av pip install scikit-image==0.18.2 pip install pooch==1.5.2 # Prepare data @@ -462,7 +464,8 @@ dy_tsm(){ cur_model_path=${BENCHMARK_ROOT}/PaddleVideo cd ${cur_model_path} - pip install wget av decord lmdb + pip install -r requirements.txt + pip install wget av decord pip install scikit-image==0.18.2 pip install pooch==1.5.2 # Prepare pretrained modles From 7a8a34678d3f386cf7df6ce12f47afcdd006f660 Mon Sep 17 00:00:00 2001 From: hysunflower Date: Mon, 10 Jan 2022 06:19:58 +0000 Subject: [PATCH 4/5] update submodule rules, test=document_fix --- scripts/auto_run_paddle.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/auto_run_paddle.sh b/scripts/auto_run_paddle.sh index 30265f0e0f..2db988775f 100644 --- a/scripts/auto_run_paddle.sh +++ b/scripts/auto_run_paddle.sh @@ -57,7 +57,7 @@ function prepare(){ apt-get install libmysqlclient20=5.7.33-0ubuntu0.16.04.1 --allow-downgrades apt-get install libmysqlclient-dev git curl psmisc -y pip install MySQL-python - + pip install shyaml save_log_dir=${all_path}/logs/${paddle_version}/${implement_type} @@ -86,12 +86,21 @@ function prepare(){ cd ${ROOT_PATH} rm -rf * - git clone https://github.com/PaddlePaddle/benchmark.git --recursive - echo "****************${implement_type} prepare had done*****************" - + git clone https://github.com/PaddlePaddle/benchmark.git cd ${BENCHMARK_ROOT} benchmark_commit_id=$(git log|head -n1|awk '{print $2}') echo "benchmark_commit_id is: "${benchmark_commit_id} + + init_group="paddle_group" # 可配到任务参数里 + repo_list=`cat submodule.yaml | shyaml get-value ${init_group}` + echo $repo_list + for i in ${repo_list[@]} + do + git submodule init $i + git submodule update $i + done + echo "*******************init submodule done******************************" + echo "****************${implement_type} prepare had done*****************" # 动态图升级到cuda10.1 python3.7，静态图切cuda10.1 python3.7 if [[ 'dynamic_graph' == ${implement_type} ]] || [[ 'static_graph' == ${implement_type} ]] || [[ 'dynamic_to_static' == ${implement_type} ]]; then From 403e5aa118d029bded6c45e66142c3efa571cf8d Mon Sep 17 00:00:00 2001 From: hysunflower Date: Mon, 10 Jan 2022 06:24:28 +0000 Subject: [PATCH 5/5] update submodule rules, test=document_fix --- submodule.yaml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 submodule.yaml diff --git a/submodule.yaml b/submodule.yaml new file mode 100644 index 0000000000..3288e6ae41 --- /dev/null +++ b/submodule.yaml @@ -0,0 +1,34 @@ +paddle_group: + models + PaddleClas + PaddleDetection + PaddleSeg + PaddleSpeech + PaddleOCR + PaddleNLP + PaddleVideo + PaddleGAN +other_frame_group: + OtherFrame/video/PyTorch/models/TimeSformer + OtherFrame/nlp/PyTorch/scripts/NLP/gpt/Megatron-LM + OtherFrame/nlp/PyTorch/apex + OtherFrame/Speech/PyTorch/wenet/models/wenet + OtherFrame/seg/PyTorch/models/mmseg + OtherFrame/clas/mxnet/models/gluon-cv + OtherFrame/clas/PyTorch/models/mmclassification + OtherFrame/clas/PyTorch/models/HRNet-Image-Classification + OtherFrame/clas/PyTorch/models/Twins + OtherFrame/gan/PyTorch/mmedting/models/mmedi + OtherFrame/ocr/PyTorch/models/PSENet + OtherFrame/nlp/PyTorch/models/xlnet/transformers + OtherFrame/detection/PyTorch/models/jde + OtherFrame/detection/PyTorch/models/fairmot + OtherFrame/Speech/PyTorch/PWGAN/models/Parakeet/PWGAN/ParallelWaveGAN + OtherFrame/detection/PyTorch/models/mmdetection + OtherFrame/detection/PyTorch/models/SOLO + OtherFrame/detection/PyTorch/models/mmpose + OtherFrame/nlp/PyTorch/models/xlnet + OtherFrame/gan/PyTorch/fomm/models/fomm + OtherFrame/gan/PyTorch/styleganv2/models/styleganv2 + OtherFrame/ocr/PyTorch/models/DB + OtherFrame/ocr/TensorFlow/models/EAST