@@ -561,6 +561,195 @@ function bind_test() {
561
561
wait
562
562
}
563
563
564
+ function parallel_test() {
565
+ mkdir -p ${PADDLE_ROOT} /build
566
+ cd ${PADDLE_ROOT} /build
567
+ if [ ${WITH_TESTING:- ON} == " ON" ] ; then
568
+ cat << EOF
569
+ ========================================
570
+ Running unit tests ...
571
+ ========================================
572
+ EOF
573
+
574
+ # calculate and set the memory usage for each process
575
+ # MEM_USAGE=$(printf "%.2f" `echo "scale=5; 1.0 / $NUM_PROC" | bc`)
576
+ # export FLAGS_fraction_of_gpu_memory_to_use=$MEM_USAGE
577
+
578
+ EXIT_CODE=0;
579
+ pids=()
580
+
581
+ # get the CUDA device count
582
+ CUDA_DEVICE_COUNT=$( nvidia-smi -L | wc -l)
583
+ # each test case would occupy two graph cards
584
+ NUM_PROC=$[CUDA_DEVICE_COUNT/2]
585
+ for (( i = 0 ; i < $NUM_PROC ; i++ )) ; do
586
+ # CUDA_VISIBLE_DEVICES http://acceleware.com/blog/cudavisibledevices-masking-gpus
587
+ # ctest -I https://cmake.org/cmake/help/v3.0/manual/ctest.1.html?highlight=ctest
588
+ if [ ${TESTING_DEBUG_MODE:- OFF} == " ON" ] ; then
589
+ env CUDA_VISIBLE_DEVICES=$[i* 2],$[i* 2+1] ctest -I $i ,,$NUM_PROC -V &
590
+ pids+=($! )
591
+ else
592
+ env CUDA_VISIBLE_DEVICES=$[i* 2],$[i* 2+1] ctest -I $i ,,$NUM_PROC --output-on-failure &
593
+ pids+=($! )
594
+ fi
595
+ done
596
+
597
+ clen=` expr " ${# pids[@]} " - 1` # get length of commands - 1
598
+ for i in ` seq 0 " $clen " ` ; do
599
+ wait ${pids[$i]}
600
+ CODE=$?
601
+ if [[ " ${CODE} " != " 0" ]]; then
602
+ echo " At least one test failed with exit code => ${CODE} " ;
603
+ EXIT_CODE=1;
604
+ fi
605
+ done
606
+ wait ; # wait for all subshells to finish
607
+
608
+ echo " EXIT_CODE => $EXIT_CODE "
609
+ if [[ " ${EXIT_CODE} " != " 0" ]]; then
610
+ exit " $EXIT_CODE "
611
+ fi
612
+ fi
613
+ }
614
+
615
+ EXIT_CODE=0;
616
+ function caught_error() {
617
+ for job in ` jobs -p` ; do
618
+ # echo "PID => ${job}"
619
+ if ! wait ${job} ; then
620
+ echo " At least one test failed with exit code => $? " ;
621
+ EXIT_CODE=1;
622
+ fi
623
+ done
624
+ }
625
+
626
+ function card_test() {
627
+ set -m
628
+
629
+ # get the CUDA device count
630
+ CUDA_DEVICE_COUNT=$( nvidia-smi -L | wc -l)
631
+
632
+ testcases=$1
633
+ if (( $# > 1 )) ; then
634
+ cardnumber=$2
635
+ if (( $cardnumber > $CUDA_DEVICE_COUNT )) ; then
636
+ cardnumber=$CUDA_DEVICE_COUNT
637
+ fi
638
+ else
639
+ cardnumber=$CUDA_DEVICE_COUNT
640
+ fi
641
+
642
+ if [[ " $testcases " == " " ]]; then
643
+ return 0
644
+ fi
645
+
646
+ trap ' caught_error' CHLD
647
+
648
+ NUM_PROC=$[CUDA_DEVICE_COUNT/$cardnumber ]
649
+ for (( i = 0 ; i < $NUM_PROC ; i++ )) ; do
650
+ # CUDA_VISIBLE_DEVICES http://acceleware.com/blog/cudavisibledevices-masking-gpus
651
+ # ctest -I https://cmake.org/cmake/help/v3.0/manual/ctest.1.html?highlight=ctest
652
+ cuda_list=()
653
+ for (( j = 0 ; j < cardnumber; j++ )) ; do
654
+ if [ $j -eq 0 ]; then
655
+ cuda_list=(" $[i*cardnumber]" )
656
+ else
657
+ cuda_list=" $cuda_list ,$[i*cardnumber+j]"
658
+ fi
659
+ done
660
+ # echo $cuda_list
661
+ if [ ${TESTING_DEBUG_MODE:- OFF} == " ON" ] ; then
662
+ if [[ $cardnumber == $CUDA_DEVICE_COUNT ]]; then
663
+ ctest -I $i ,,$NUM_PROC -R " ($testcases )" -V &
664
+ else
665
+ env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i ,,$NUM_PROC -R " ($testcases )" -V &
666
+ fi
667
+ else
668
+ if [[ $cardnumber == $CUDA_DEVICE_COUNT ]]; then
669
+ ctest -I $i ,,$NUM_PROC -R " ($testcases )" --output-on-failure &
670
+ else
671
+ # echo "env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i,,$NUM_PROC -R \"($testcases)\" --output-on-failure &"
672
+ env CUDA_VISIBLE_DEVICES=$cuda_list ctest -I $i ,,$NUM_PROC -R " ($testcases )" --output-on-failure &
673
+ fi
674
+ fi
675
+ done
676
+
677
+ wait ; # wait for all subshells to finish
678
+ }
679
+
680
+ function aggresive_test() {
681
+ mkdir -p ${PADDLE_ROOT} /build
682
+ cd ${PADDLE_ROOT} /build
683
+ if [ ${WITH_TESTING:- ON} == " ON" ] ; then
684
+ cat << EOF
685
+ ========================================
686
+ Running unit tests ...
687
+ ========================================
688
+ EOF
689
+
690
+ set +x
691
+ EXIT_CODE=0;
692
+ test_cases=$( ctest -N -V)
693
+ exclusive_tests=' '
694
+ single_card_tests=' '
695
+ multiple_card_tests=' '
696
+ is_exclusive=' '
697
+ is_multicard=' '
698
+ while read -r line; do
699
+ if [[ " $line " == " " ]]; then
700
+ continue
701
+ fi
702
+ read matchstr <<< $( echo " $line " | grep -oEi ' Test[ \t]+#' )
703
+ if [[ " $matchstr " == " " ]]; then
704
+ # Any test case with LABELS property would be parse here
705
+ # RUN_TYPE=EXCLUSIVE mean the case would run exclusively
706
+ # RUN_TYPE=DIST mean the case would take two graph cards during runtime
707
+ read is_exclusive <<< $( echo " $line " | grep -oEi " RUN_TYPE=EXCLUSIVE" )
708
+ read is_multicard <<< $( echo " $line " | grep -oEi " RUN_TYPE=DIST" )
709
+ continue
710
+ fi
711
+ read testcase <<< $( echo " $line " | grep -oEi " \w+$" )
712
+
713
+ if [[ " $is_multicard " == " " ]]; then
714
+ # trick: treat all test case with prefix "test_dist" as dist case, and would run on 2 cards
715
+ read is_multicard <<< $( echo " $testcase " | grep -oEi " test_dist" )
716
+ fi
717
+
718
+ if [[ " $is_exclusive " != " " ]]; then
719
+ if [[ " $exclusive_tests " == " " ]]; then
720
+ exclusive_tests=" ^$testcase $"
721
+ else
722
+ exclusive_tests=" $exclusive_tests |^$testcase $"
723
+ fi
724
+ elif [[ " $is_multicard " != " " ]]; then
725
+ if [[ " $multiple_card_tests " == " " ]]; then
726
+ multiple_card_tests=" ^$testcase $"
727
+ else
728
+ multiple_card_tests=" $multiple_card_tests |^$testcase $"
729
+ fi
730
+ else
731
+ if [[ " $single_card_tests " == " " ]]; then
732
+ single_card_tests=" ^$testcase $"
733
+ else
734
+ single_card_tests=" $single_card_tests |^$testcase $"
735
+ fi
736
+ fi
737
+ is_exclusive=' '
738
+ is_multicard=' '
739
+ matchstr=' '
740
+ testcase=' '
741
+ done <<< " $test_cases" ;
742
+
743
+ card_test " $single_card_tests " 1
744
+ card_test " $multiple_card_tests " 2
745
+ card_test " $exclusive_tests "
746
+ if [[ " $EXIT_CODE " != " 0" ]]; then
747
+ exit 1;
748
+ fi
749
+ set -ex
750
+ fi
751
+ }
752
+
564
753
function gen_doc_lib() {
565
754
mkdir -p ${PADDLE_ROOT} /build
566
755
cd ${PADDLE_ROOT} /build
@@ -805,7 +994,7 @@ function main() {
805
994
gen_dockerfile ${PYTHON_ABI:- " " }
806
995
;;
807
996
test)
808
- run_test
997
+ aggresive_test
809
998
;;
810
999
single_test)
811
1000
single_test $2
@@ -835,7 +1024,7 @@ function main() {
835
1024
cmake_gen ${PYTHON_ABI:- " " }
836
1025
build ${parallel_number}
837
1026
assert_api_not_changed ${PYTHON_ABI:- " " }
838
- run_test
1027
+ aggresive_test
839
1028
gen_fluid_lib ${parallel_number}
840
1029
test_fluid_lib
841
1030
assert_api_spec_approvals
@@ -868,7 +1057,7 @@ function main() {
868
1057
cicheck_py35)
869
1058
cmake_gen ${PYTHON_ABI:- " " }
870
1059
build ${parallel_number}
871
- run_test
1060
+ aggresive_test
872
1061
assert_api_not_changed ${PYTHON_ABI:- " " }
873
1062
;;
874
1063
cmake_gen)
@@ -882,7 +1071,7 @@ function main() {
882
1071
;;
883
1072
* )
884
1073
print_usage
885
- exit 0
1074
+ exit 1
886
1075
;;
887
1076
esac
888
1077
}
0 commit comments