Skip to content

Commit d9991dc

Browse files
author
wopeizl
authored
add parallel build script to ci … (PaddlePaddle#16901)
* add parallel build script to ci test=develop * 1. classify the test case as single card/two cards/multiple cards type 2. run test case according to the run type
1 parent 24923f7 commit d9991dc

File tree

6 files changed

+211
-5
lines changed

6 files changed

+211
-5
lines changed

paddle/fluid/inference/api/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ if(WITH_TESTING)
5656
inference_base_test(test_api_impl SRCS api_impl_tester.cc DEPS ${inference_deps}
5757
ARGS --word2vec_dirname=${WORD2VEC_MODEL_DIR} --book_dirname=${PYTHON_TESTS_DIR}/book)
5858
set_tests_properties(test_api_impl PROPERTIES DEPENDS test_image_classification)
59+
set_tests_properties(test_api_impl PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
5960
endif()
6061
cc_test(test_analysis_predictor SRCS analysis_predictor_tester.cc DEPS analysis_predictor benchmark ${inference_deps}
6162
ARGS --dirname=${WORD2VEC_MODEL_DIR})

paddle/fluid/inference/tests/book/CMakeLists.txt

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@ function(inference_test TARGET_NAME)
1919
DEPS paddle_fluid_origin
2020
ARGS --dirname=${PYTHON_TESTS_DIR}/book/${TARGET_NAME}${arg}.inference.model)
2121
set_tests_properties(test_inference_${TARGET_NAME}${arg}
22-
PROPERTIES DEPENDS test_${TARGET_NAME})
22+
PROPERTIES DEPENDS test_${TARGET_NAME})
23+
set_tests_properties(test_inference_${TARGET_NAME}${arg}
24+
PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
2325
endforeach()
2426
endfunction(inference_test)
2527

@@ -45,3 +47,4 @@ cc_test(test_inference_nlp
4547
DEPS paddle_fluid_origin
4648
ARGS
4749
--model_path=${PADDLE_BINARY_DIR}/python/paddle/fluid/tests/book/recognize_digits_mlp.inference.model)
50+
set_tests_properties(test_inference_nlp PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")

paddle/fluid/train/CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ function(train_test TARGET_NAME)
2020
ARGS --dirname=${PYTHON_TESTS_DIR}/book/${TARGET_NAME}${arg}.train.model/)
2121
set_tests_properties(test_train_${TARGET_NAME}${arg}
2222
PROPERTIES DEPENDS test_${TARGET_NAME})
23+
set_tests_properties(test_train_${TARGET_NAME}${arg}
24+
PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
2325
endforeach()
2426
endfunction(train_test)
2527

paddle/scripts/paddle_build.sh

Lines changed: 193 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -561,6 +561,195 @@ function bind_test() {
561561
wait
562562
}
563563

564+
function parallel_test() {
565+
mkdir -p ${PADDLE_ROOT}/build
566+
cd ${PADDLE_ROOT}/build
567+
if [ ${WITH_TESTING:-ON} == "ON" ] ; then
568+
cat <<EOF
569+
========================================
570+
Running unit tests ...
571+
========================================
572+
EOF
573+
574+
# calculate and set the memory usage for each process
575+
# MEM_USAGE=$(printf "%.2f" `echo "scale=5; 1.0 / $NUM_PROC" | bc`)
576+
# export FLAGS_fraction_of_gpu_memory_to_use=$MEM_USAGE
577+
578+
EXIT_CODE=0;
579+
pids=()
580+
581+
# get the CUDA device count
582+
CUDA_DEVICE_COUNT=$(nvidia-smi -L | wc -l)
583+
# each test case would occupy two graph cards
584+
NUM_PROC=$[CUDA_DEVICE_COUNT/2]
585+
for (( i = 0; i < $NUM_PROC; i++ )); do
586+
# CUDA_VISIBLE_DEVICES http://acceleware.com/blog/cudavisibledevices-masking-gpus
587+
# ctest -I https://cmake.org/cmake/help/v3.0/manual/ctest.1.html?highlight=ctest
588+
if [ ${TESTING_DEBUG_MODE:-OFF} == "ON" ] ; then
589+
env CUDA_VISIBLE_DEVICES=$[i*2],$[i*2+1] ctest -I $i,,$NUM_PROC -V &
590+
pids+=($!)
591+
else
592+
env CUDA_VISIBLE_DEVICES=$[i*2],$[i*2+1] ctest -I $i,,$NUM_PROC --output-on-failure &
593+
pids+=($!)
594+
fi
595+
done
596+
597+
clen=`expr "${#pids[@]}" - 1` # get length of commands - 1
598+
for i in `seq 0 "$clen"`; do
599+
wait ${pids[$i]}
600+
CODE=$?
601+
if [[ "${CODE}" != "0" ]]; then
602+
echo "At least one test failed with exit code => ${CODE}" ;
603+
EXIT_CODE=1;
604+
fi
605+
done
606+
wait; # wait for all subshells to finish
607+
608+
echo "EXIT_CODE => $EXIT_CODE"
609+
if [[ "${EXIT_CODE}" != "0" ]]; then
610+
exit "$EXIT_CODE"
611+
fi
612+
fi
613+
}
614+
615+
EXIT_CODE=0;
616+
function caught_error() {
617+
for job in `jobs -p`; do
618+
# echo "PID => ${job}"
619+
if ! wait ${job} ; then
620+
echo "At least one test failed with exit code => $?" ;
621+
EXIT_CODE=1;
622+
fi
623+
done
624+
}
625+
626+
function card_test() {
627+
set -m
628+
629+
# get the CUDA device count
630+
CUDA_DEVICE_COUNT=$(nvidia-smi -L | wc -l)
631+
632+
testcases=$1
633+
if (( $# > 1 )); then
634+
cardnumber=$2
635+
if (( $cardnumber > $CUDA_DEVICE_COUNT )); then
636+
cardnumber=$CUDA_DEVICE_COUNT
637+
fi
638+
else
639+
cardnumber=$CUDA_DEVICE_COUNT
640+
fi
641+
642+
if [[ "$testcases" == "" ]]; then
643+
return 0
644+
fi
645+
646+
trap 'caught_error' CHLD
647+
648+
NUM_PROC=$[CUDA_DEVICE_COUNT/$cardnumber]
649+
for (( i = 0; i < $NUM_PROC; i++ )); do
650+
# CUDA_VISIBLE_DEVICES http://acceleware.com/blog/cudavisibledevices-masking-gpus
651+
# ctest -I https://cmake.org/cmake/help/v3.0/manual/ctest.1.html?highlight=ctest
652+
cuda_list=()
653+
for (( j = 0; j < cardnumber; j++ )); do
654+
if [ $j -eq 0 ]; then
655+
cuda_list=("$[i*cardnumber]")
656+
else
657+
cuda_list="$cuda_list,$[i*cardnumber+j]"
658+
fi
659+
done
660+
# echo $cuda_list
661+
if [ ${TESTING_DEBUG_MODE:-OFF} == "ON" ] ; then
662+
if [[ $cardnumber == $CUDA_DEVICE_COUNT ]]; then
663+
ctest -I $i,,$NUM_PROC -R "($testcases)" -V &
664+
else
665+
env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC -R "($testcases)" -V &
666+
fi
667+
else
668+
if [[ $cardnumber == $CUDA_DEVICE_COUNT ]]; then
669+
ctest -I $i,,$NUM_PROC -R "($testcases)" --output-on-failure &
670+
else
671+
# echo "env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC -R \"($testcases)\" --output-on-failure &"
672+
env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC -R "($testcases)" --output-on-failure &
673+
fi
674+
fi
675+
done
676+
677+
wait; # wait for all subshells to finish
678+
}
679+
680+
function aggresive_test() {
681+
mkdir -p ${PADDLE_ROOT}/build
682+
cd ${PADDLE_ROOT}/build
683+
if [ ${WITH_TESTING:-ON} == "ON" ] ; then
684+
cat <<EOF
685+
========================================
686+
Running unit tests ...
687+
========================================
688+
EOF
689+
690+
set +x
691+
EXIT_CODE=0;
692+
test_cases=$(ctest -N -V)
693+
exclusive_tests=''
694+
single_card_tests=''
695+
multiple_card_tests=''
696+
is_exclusive=''
697+
is_multicard=''
698+
while read -r line; do
699+
if [[ "$line" == "" ]]; then
700+
continue
701+
fi
702+
read matchstr <<< $(echo "$line"|grep -oEi 'Test[ \t]+#')
703+
if [[ "$matchstr" == "" ]]; then
704+
# Any test case with LABELS property would be parse here
705+
# RUN_TYPE=EXCLUSIVE mean the case would run exclusively
706+
# RUN_TYPE=DIST mean the case would take two graph cards during runtime
707+
read is_exclusive <<< $(echo "$line"|grep -oEi "RUN_TYPE=EXCLUSIVE")
708+
read is_multicard <<< $(echo "$line"|grep -oEi "RUN_TYPE=DIST")
709+
continue
710+
fi
711+
read testcase <<< $(echo "$line"|grep -oEi "\w+$")
712+
713+
if [[ "$is_multicard" == "" ]]; then
714+
# trick: treat all test case with prefix "test_dist" as dist case, and would run on 2 cards
715+
read is_multicard <<< $(echo "$testcase"|grep -oEi "test_dist")
716+
fi
717+
718+
if [[ "$is_exclusive" != "" ]]; then
719+
if [[ "$exclusive_tests" == "" ]]; then
720+
exclusive_tests="^$testcase$"
721+
else
722+
exclusive_tests="$exclusive_tests|^$testcase$"
723+
fi
724+
elif [[ "$is_multicard" != "" ]]; then
725+
if [[ "$multiple_card_tests" == "" ]]; then
726+
multiple_card_tests="^$testcase$"
727+
else
728+
multiple_card_tests="$multiple_card_tests|^$testcase$"
729+
fi
730+
else
731+
if [[ "$single_card_tests" == "" ]]; then
732+
single_card_tests="^$testcase$"
733+
else
734+
single_card_tests="$single_card_tests|^$testcase$"
735+
fi
736+
fi
737+
is_exclusive=''
738+
is_multicard=''
739+
matchstr=''
740+
testcase=''
741+
done <<< "$test_cases";
742+
743+
card_test "$single_card_tests" 1
744+
card_test "$multiple_card_tests" 2
745+
card_test "$exclusive_tests"
746+
if [[ "$EXIT_CODE" != "0" ]]; then
747+
exit 1;
748+
fi
749+
set -ex
750+
fi
751+
}
752+
564753
function gen_doc_lib() {
565754
mkdir -p ${PADDLE_ROOT}/build
566755
cd ${PADDLE_ROOT}/build
@@ -805,7 +994,7 @@ function main() {
805994
gen_dockerfile ${PYTHON_ABI:-""}
806995
;;
807996
test)
808-
run_test
997+
aggresive_test
809998
;;
810999
single_test)
8111000
single_test $2
@@ -835,7 +1024,7 @@ function main() {
8351024
cmake_gen ${PYTHON_ABI:-""}
8361025
build ${parallel_number}
8371026
assert_api_not_changed ${PYTHON_ABI:-""}
838-
run_test
1027+
aggresive_test
8391028
gen_fluid_lib ${parallel_number}
8401029
test_fluid_lib
8411030
assert_api_spec_approvals
@@ -868,7 +1057,7 @@ function main() {
8681057
cicheck_py35)
8691058
cmake_gen ${PYTHON_ABI:-""}
8701059
build ${parallel_number}
871-
run_test
1060+
aggresive_test
8721061
assert_api_not_changed ${PYTHON_ABI:-""}
8731062
;;
8741063
cmake_gen)
@@ -882,7 +1071,7 @@ function main() {
8821071
;;
8831072
*)
8841073
print_usage
885-
exit 0
1074+
exit 1
8861075
;;
8871076
esac
8881077
}

python/paddle/fluid/tests/book/high-level-api/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ else()
2323
message(WARNING "These tests has been disabled in OSX for random fail: \n" ${src})
2424
elseif()
2525
py_test(${src} SRCS ${src}.py)
26+
set_tests_properties(${src} PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
2627
endif()
2728
endforeach()
2829
endif()

python/paddle/fluid/tests/unittests/CMakeLists.txt

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,3 +137,13 @@ endif()
137137
if (WITH_MKLDNN)
138138
add_subdirectory(mkldnn)
139139
endif()
140+
141+
if(WITH_DISTRIBUTE)
142+
set_tests_properties(test_listen_and_serv_op test_nce_remote_table_op test_hsigmoid_remote_table_op
143+
PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
144+
endif()
145+
146+
set_tests_properties(test_weight_decay test_conv_shift_op test_alloc_continuous_space_op test_recordio_reader
147+
test_parallel_executor_test_while_train test_adam_op_multi_thread test_parallel_executor_mnist
148+
test_parallel_executor_seresnext test_parallel_executor_crf test_nearest_interp_op
149+
PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")

0 commit comments

Comments
 (0)