diff --git a/PaddleNLP b/PaddleNLP index b15382b9b7..719485c674 160000 --- a/PaddleNLP +++ b/PaddleNLP @@ -1 +1 @@ -Subproject commit b15382b9b7bfd6c5043930ca6667ad84e8f49653 +Subproject commit 719485c674ca94dc142bbaa23b2c7777ca152f4b diff --git a/scripts/auto_run_paddle.sh b/scripts/auto_run_paddle.sh index 30265f0e0f..2db988775f 100644 --- a/scripts/auto_run_paddle.sh +++ b/scripts/auto_run_paddle.sh @@ -57,7 +57,7 @@ function prepare(){ apt-get install libmysqlclient20=5.7.33-0ubuntu0.16.04.1 --allow-downgrades apt-get install libmysqlclient-dev git curl psmisc -y pip install MySQL-python - + pip install shyaml save_log_dir=${all_path}/logs/${paddle_version}/${implement_type} @@ -86,12 +86,21 @@ function prepare(){ cd ${ROOT_PATH} rm -rf * - git clone https://github.com/PaddlePaddle/benchmark.git --recursive - echo "****************${implement_type} prepare had done*****************" - + git clone https://github.com/PaddlePaddle/benchmark.git cd ${BENCHMARK_ROOT} benchmark_commit_id=$(git log|head -n1|awk '{print $2}') echo "benchmark_commit_id is: "${benchmark_commit_id} + + init_group="paddle_group" # 可配到任务参数里 + repo_list=`cat submodule.yaml | shyaml get-value ${init_group}` + echo $repo_list + for i in ${repo_list[@]} + do + git submodule init $i + git submodule update $i + done + echo "*******************init submodule done******************************" + echo "****************${implement_type} prepare had done*****************" # 动态图升级到cuda10.1 python3.7,静态图切cuda10.1 python3.7 if [[ 'dynamic_graph' == ${implement_type} ]] || [[ 'static_graph' == ${implement_type} ]] || [[ 'dynamic_to_static' == ${implement_type} ]]; then diff --git a/scripts/dynamic_graph_models.sh b/scripts/dynamic_graph_models.sh index d8fde7bb2e..5e98490a9d 100644 --- a/scripts/dynamic_graph_models.sh +++ b/scripts/dynamic_graph_models.sh @@ -72,9 +72,10 @@ dy_video_TimeSformer(){ echo "dy_video_TimeSformer" cur_model_path=${BENCHMARK_ROOT}/PaddleVideo/ cd ${cur_model_path}/benchmark/TimeSformer/ + pip install -r requirements.txt pip install scikit-image==0.18.2 pip install pooch==1.5.2 - bash run_all.sh local + bash run_all.sh local rm -rf ${BENCHMARK_ROOT}/PaddleVideo/ # 避免数据集占用docker内过多空间,在执行最后一个模型后删掉 } @@ -256,6 +257,7 @@ dy_tsn(){ cur_model_path=${BENCHMARK_ROOT}/PaddleVideo cd ${cur_model_path} + pip install -r requirements.txt pip install wget av pip install scikit-image==0.18.2 pip install pooch==1.5.2 @@ -345,6 +347,7 @@ dy_seg(){ dy_slowfast(){ cur_model_path=${BENCHMARK_ROOT}/PaddleVideo cd ${cur_model_path} + pip install -r requirements.txt pip install tqdm pip install decord pip install pandas av @@ -461,6 +464,7 @@ dy_tsm(){ cur_model_path=${BENCHMARK_ROOT}/PaddleVideo cd ${cur_model_path} + pip install -r requirements.txt pip install wget av decord pip install scikit-image==0.18.2 pip install pooch==1.5.2 diff --git a/submodule.yaml b/submodule.yaml new file mode 100644 index 0000000000..3288e6ae41 --- /dev/null +++ b/submodule.yaml @@ -0,0 +1,34 @@ +paddle_group: + models + PaddleClas + PaddleDetection + PaddleSeg + PaddleSpeech + PaddleOCR + PaddleNLP + PaddleVideo + PaddleGAN +other_frame_group: + OtherFrame/video/PyTorch/models/TimeSformer + OtherFrame/nlp/PyTorch/scripts/NLP/gpt/Megatron-LM + OtherFrame/nlp/PyTorch/apex + OtherFrame/Speech/PyTorch/wenet/models/wenet + OtherFrame/seg/PyTorch/models/mmseg + OtherFrame/clas/mxnet/models/gluon-cv + OtherFrame/clas/PyTorch/models/mmclassification + OtherFrame/clas/PyTorch/models/HRNet-Image-Classification + OtherFrame/clas/PyTorch/models/Twins + OtherFrame/gan/PyTorch/mmedting/models/mmedi + OtherFrame/ocr/PyTorch/models/PSENet + OtherFrame/nlp/PyTorch/models/xlnet/transformers + OtherFrame/detection/PyTorch/models/jde + OtherFrame/detection/PyTorch/models/fairmot + OtherFrame/Speech/PyTorch/PWGAN/models/Parakeet/PWGAN/ParallelWaveGAN + OtherFrame/detection/PyTorch/models/mmdetection + OtherFrame/detection/PyTorch/models/SOLO + OtherFrame/detection/PyTorch/models/mmpose + OtherFrame/nlp/PyTorch/models/xlnet + OtherFrame/gan/PyTorch/fomm/models/fomm + OtherFrame/gan/PyTorch/styleganv2/models/styleganv2 + OtherFrame/ocr/PyTorch/models/DB + OtherFrame/ocr/TensorFlow/models/EAST