[CI] Add Slice #380
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Coverage | |
| on: | |
| pull_request: | |
| types: [opened, synchronize] | |
| branches: [develop, release/**] | |
| permissions: read-all | |
| concurrency: | |
| group: ${{ github.event.pull_request.number }}-${{ github.workflow }} | |
| cancel-in-progress: true | |
| env: | |
| Dockerfile: Dockerfile.cuda117_cudnn8_gcc82_ubuntu18_coverage | |
| docker_image: 5a60bb165c34 | |
| PR_ID: ${{ github.event.pull_request.number }} | |
| COMMIT_ID: ${{ github.event.pull_request.head.sha }} | |
| TASK: paddle-CI-${{ github.event.pull_request.number }}-coverage | |
| ci_scripts: /paddle/ci | |
| BRANCH: ${{ github.base_ref }} | |
| work_dir: /paddle | |
| PADDLE_ROOT: /paddle | |
| GIT_PR_ID: ${{ github.event.pull_request.number }} | |
| CI_name: coverage | |
| no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn" | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| clone: | |
| name: Coverage clone | |
| uses: ./.github/workflows/_Clone-linux.yml | |
| with: | |
| workflow-name: 'coverage' | |
| bos_dir: Paddle-coverage | |
| build-docker: | |
| name: Coverage build docker | |
| needs: clone | |
| uses: ./.github/workflows/docker.yml | |
| with: | |
| bos_dir: Paddle-coverage | |
| build: | |
| name: Coverage build | |
| needs: [clone, build-docker] | |
| if: needs.clone.outputs.can-skip != 'true' | |
| runs-on: | |
| group: GZ_BD-CPU | |
| outputs: | |
| can-skip: ${{ steps.check-bypass.outputs.can-skip }} | |
| steps: | |
| - name: Check docker image and run container | |
| if: steps.check-bypass.outputs.can-skip != 'true' | |
| env: | |
| CCACHE_DIR: "/root/.ccache/coverage" | |
| FLAGS_fraction_of_gpu_memory_to_use: 0.15 | |
| CTEST_PARALLEL_LEVEL: 2 | |
| WITH_GPU: "ON" | |
| CUDA_ARCH_NAME: Volta | |
| WITH_AVX: "ON" | |
| WITH_COVERAGE: "ON" | |
| COVERALLS_UPLOAD: "ON" | |
| PADDLE_VERSION: 0.0.0 | |
| CUDA_VISIBLE_DEVICES: 0,1 | |
| WITH_DISTRIBUTE: "ON" | |
| PRECISION_TEST: "ON" | |
| LITE_GIT_TAG: develop | |
| WITH_UNITY_BUILD: "ON" | |
| PY_VERSION: 3.9 | |
| WITH_SHARED_PHI: "ON" | |
| WITH_CINN: "ON" | |
| INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage | |
| CCACHE_MAXSIZE: 200G | |
| CCACHE_LIMIT_MULTIPLE: 0.8 | |
| ON_INFER: "ON" | |
| PADDLE_CUDA_INSTALL_REQUIREMENTS: "ON" | |
| FLAGS_enable_unused_var_check: 1 | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| UT_RUN_TYPE_SETTING: WITHOUT_HYBRID | |
| run: | | |
| container_name=${TASK}-build-$(date +%Y%m%d-%H%M%S) | |
| echo "container_name=${container_name}" >> ${{ github.env }} | |
| docker_image=${{ needs.build-docker.outputs.docker_coverage_image }} | |
| docker run -d -t --name ${container_name} \ | |
| -v "/home/data/cfs:/home/data/cfs" \ | |
| -v "/home/data/cfs/.cache:/root/.cache" \ | |
| -v "/home/data/cfs/.ccache:/root/.ccache" \ | |
| -v "/dev/shm:/dev/shm" \ | |
| -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \ | |
| -v ${{ github.workspace }}:/paddle \ | |
| -e CI_name \ | |
| -e BRANCH \ | |
| -e PR_ID \ | |
| -e COMMIT_ID \ | |
| -e work_dir \ | |
| -e PADDLE_ROOT \ | |
| -e GIT_PR_ID \ | |
| -e CCACHE_DIR \ | |
| -e ci_scripts \ | |
| -e FLAGS_fraction_of_gpu_memory_to_use \ | |
| -e CTEST_PARALLEL_LEVEL \ | |
| -e WITH_GPU \ | |
| -e CUDA_ARCH_NAME \ | |
| -e WITH_AVX \ | |
| -e WITH_COVERAGE \ | |
| -e COVERALLS_UPLOAD \ | |
| -e PADDLE_VERSION \ | |
| -e WITH_DISTRIBUTE \ | |
| -e PRECISION_TEST \ | |
| -e LITE_GIT_TAG \ | |
| -e WITH_UNITY_BUILD \ | |
| -e PY_VERSION \ | |
| -e WITH_SHARED_PHI \ | |
| -e WITH_CINN \ | |
| -e INFERENCE_DEMO_INSTALL_DIR \ | |
| -e CCACHE_MAXSIZE \ | |
| -e CCACHE_LIMIT_MULTIPLE \ | |
| -e ON_INFER \ | |
| -e PADDLE_CUDA_INSTALL_REQUIREMENTS \ | |
| -e FLAGS_enable_unused_var_check \ | |
| -e GITHUB_TOKEN \ | |
| -e GITHUB_API_TOKEN \ | |
| -e UT_RUN_TYPE_SETTING \ | |
| -e no_proxy \ | |
| -w /paddle --network host ${docker_image} | |
| - name: Download paddle.tar.gz and update test branch | |
| run: | | |
| docker exec -t ${{ env.container_name }} /bin/bash -c ' | |
| rm -rf * .[^.]* | |
| set -e | |
| echo "Downloading Paddle.tar.gz" | |
| wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle-coverage/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate | |
| echo "Extracting Paddle.tar.gz" | |
| tar xf Paddle.tar.gz --strip-components=1 | |
| rm Paddle.tar.gz | |
| git remote add upstream https://github.com/PaddlePaddle/Paddle.git | |
| git config pull.rebase false | |
| git checkout test | |
| echo "Pull upstream $BRANCH" | |
| source ${{ github.workspace }}/../../../proxy | |
| git pull upstream $BRANCH --no-edit | |
| ' | |
| - name: Check bypass | |
| id: check-bypass | |
| uses: ./.github/actions/check-bypass | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| workflow-name: coverage | |
| - name: Build | |
| if: steps.check-bypass.outputs.can-skip != 'true' | |
| run: | | |
| docker exec -t ${{ env.container_name }} /bin/bash -c ' | |
| mkdir -p /home/data/cfs/.cache/coverage | |
| mkdir -p /home/data/cfs/.ccache/coverage | |
| source ${{ github.workspace }}/../../../proxy | |
| bash $ci_scripts/coverage_build.sh bdist_wheel | |
| ' | |
| - name: Check added unittests | |
| if: steps.check-bypass.outputs.can-skip != 'true' | |
| run: | | |
| docker exec -t ${{ env.container_name }} /bin/bash -c ' | |
| source ~/.bashrc | |
| source ${{ github.workspace }}/../../../proxy | |
| bash $ci_scripts/check_added_ut.sh | |
| ' | |
| - name: Check coverage build size requires approval | |
| if: steps.check-bypass.outputs.can-skip != 'true' | |
| run: | | |
| docker exec -t ${{ env.container_name }} /bin/bash -c ' | |
| source ~/.bashrc | |
| source ${{ github.workspace }}/../../../proxy | |
| bash $ci_scripts/coverage_build_size_approval.sh | |
| ' | |
| - name: Clean up env | |
| if: steps.check-bypass.outputs.can-skip != 'true' | |
| run: | | |
| docker exec -t ${{ env.container_name }} /bin/bash -c ' | |
| source ~/.bashrc | |
| source ${ci_scripts}/utils.sh; clean_build_files | |
| Build_Size=$(du -h --max-depth=0 ${work_dir}/build |awk '"'"'{print $1}'"'"') | |
| echo "Build_Size=${Build_Size}" > ${work_dir}/dist/coverage_build_size | |
| find ./ -type f -size +200M | xargs du -lh | |
| rm -rf $(find . -name "*.a") | |
| rm -rf $(find . -name "*.o") | |
| rm -rf paddle_inference_install_dir | |
| rm -rf paddle_inference_c_install_dir | |
| rm -rf lib.linux-x86_64-3.9 | |
| find ./ -name "eager_generator" -or -name "kernel_signature_generator" -or -name "eager_legacy_op_function_generator" | xargs rm -rf | |
| rm -rf ./python/build/lib.linux-x86_64-3.9/ | |
| cd "${work_dir}/build/third_party" && find $(ls | grep -v "dlpack" | grep -v "install" | grep -v "eigen3" | grep -v "gflags") -type f ! -name "*.so" -a ! -name "libdnnl.so*" -delete | |
| cd / | |
| tar --use-compress-program="pzstd -1" -cf Paddle.tar.gz paddle | |
| ' | |
| - name: Upload coverage product | |
| if: steps.check-bypass.outputs.can-skip != 'true' | |
| env: | |
| home_path: ${{ github.workspace }}/.. | |
| bos_file: ${{ github.workspace }}/../bos/BosClient.py | |
| paddle_whl: paddlepaddle_gpu-0.0.0-cp39-cp39-linux_x86_64.whl | |
| run: | | |
| docker exec -t ${{ env.container_name }} /bin/bash -c ' | |
| echo "::group::Install bce-python-sdk" | |
| source ${{ github.workspace }}/../../../proxy | |
| python -m pip install bce-python-sdk==0.8.74 | |
| echo "::endgroup::" | |
| export AK=paddle | |
| export SK=paddle | |
| if [ ! -f "${{ env.bos_file }}" ]; then | |
| wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate | |
| mkdir ${{ env.home_path }}/bos | |
| tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos | |
| fi | |
| cd /paddle/dist | |
| echo "Uploading coverage build size" | |
| python ${{ env.bos_file }} coverage_build_size paddle-github-action/PR/coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }} | |
| echo "Uploading coverage wheel" | |
| python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }} | |
| cd / | |
| echo "Uploading Paddle.tar.gz" | |
| python ${{ env.bos_file }} Paddle.tar.gz paddle-github-action/PR/coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }} | |
| rm Paddle.tar.gz | |
| ' | |
| - name: Terminate and delete the container | |
| if: ${{ steps.check-bypass.outputs.can-skip != 'true' && always() }} | |
| run: | | |
| set +e | |
| docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*' | |
| docker stop ${{ env.container_name }} | |
| docker rm ${{ env.container_name }} | |
| test: | |
| name: Coverage test | |
| needs: build | |
| if: needs.build.outputs.can-skip != 'true' | |
| runs-on: | |
| group: BD_BJ-V100 | |
| steps: | |
| - name: Check docker image and run container | |
| env: | |
| CACHE_DIR: "/root/.cache/coverage" | |
| CCACHE_DIR: "/root/.ccache/coverage" | |
| FLAGS_fraction_of_gpu_memory_to_use: 0.15 | |
| CTEST_PARALLEL_LEVEL: 2 | |
| WITH_GPU: "ON" | |
| CUDA_ARCH_NAME: Volta | |
| WITH_AVX: "ON" | |
| WITH_COVERAGE: "ON" | |
| COVERALLS_UPLOAD: "ON" | |
| PADDLE_VERSION: 0.0.0 | |
| WITH_DISTRIBUTE: "ON" | |
| PRECISION_TEST: "ON" | |
| WITH_UNITY_BUILD: "ON" | |
| PY_VERSION: 3.9 | |
| WITH_SHARED_PHI: "ON" | |
| WITH_CINN: "ON" | |
| INFERENCE_DEMO_INSTALL_DIR: /root/.cache/coverage | |
| CCACHE_MAXSIZE: 200G | |
| CCACHE_LIMIT_MULTIPLE: 0.8 | |
| FLAGS_PIR_OPTEST: "TRUE" | |
| ON_INFER: "ON" | |
| COVERAGE_FILE: ${{ github.workspace }}/build/python-coverage.data | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| container_name=${TASK}-$(date +%Y%m%d-%H%M%S) | |
| echo "container_name=${container_name}" >> ${{ github.env }} | |
| docker run -d -t --gpus all --name ${container_name} \ | |
| -v "/home/data/cfs:/home/data/cfs" \ | |
| -v "/home/data/cfs/.cache:/root/.cache" \ | |
| -v "/home/data/cfs/.ccache:/root/.ccache" \ | |
| -v "/dev/shm:/dev/shm" \ | |
| -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \ | |
| -v ${{ github.workspace }}:/paddle \ | |
| -e CI_name \ | |
| -e BRANCH \ | |
| -e PR_ID \ | |
| -e COMMIT_ID \ | |
| -e work_dir \ | |
| -e PADDLE_ROOT \ | |
| -e GIT_PR_ID \ | |
| -e CACHE_DIR \ | |
| -e CCACHE_DIR \ | |
| -e ci_scripts \ | |
| -e FLAGS_fraction_of_gpu_memory_to_use \ | |
| -e CTEST_PARALLEL_LEVEL \ | |
| -e WITH_GPU \ | |
| -e CUDA_ARCH_NAME \ | |
| -e WITH_AVX \ | |
| -e WITH_COVERAGE \ | |
| -e COVERALLS_UPLOAD \ | |
| -e PADDLE_VERSION \ | |
| -e WITH_DISTRIBUTE \ | |
| -e PRECISION_TEST \ | |
| -e WITH_UNITY_BUILD \ | |
| -e PY_VERSION \ | |
| -e WITH_SHARED_PHI \ | |
| -e WITH_CINN \ | |
| -e INFERENCE_DEMO_INSTALL_DIR \ | |
| -e CCACHE_MAXSIZE \ | |
| -e CCACHE_LIMIT_MULTIPLE \ | |
| -e FLAGS_PIR_OPTEST \ | |
| -e ON_INFER \ | |
| -e COVERAGE_FILE \ | |
| -e GITHUB_TOKEN \ | |
| -e GITHUB_API_TOKEN \ | |
| -e no_proxy \ | |
| -w /paddle --network host ${docker_image} | |
| - name: Download paddle.tar.gz and update test branch | |
| run: | | |
| docker exec -t ${{ env.container_name }} /bin/bash -c ' | |
| rm -rf * .[^.]* | |
| set -e | |
| echo "Downloading Paddle.tar.gz" | |
| wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/coverage/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate | |
| echo "Extracting Paddle.tar.gz" | |
| tar --use-compress-program="pzstd -1" -xf Paddle.tar.gz --strip-components=1 | |
| rm Paddle.tar.gz | |
| source ${{ github.workspace }}/../../../proxy | |
| git checkout test | |
| echo "Pull upstream develop" | |
| git pull upstream $BRANCH --no-edit | |
| ' | |
| - name: Test | |
| run: | | |
| docker exec -t ${{ env.container_name }} /bin/bash -c ' | |
| source ${{ github.workspace }}/../../../proxy | |
| bash $ci_scripts/coverage_test.sh | |
| TEST_EXIT_CODE=$? | |
| echo "TEST_EXIT_CODE=${TEST_EXIT_CODE}" >> ${{ github.env }} | |
| if [[ "$TEST_EXIT_CODE" -ne 0 && "$TEST_EXIT_CODE" -ne 9 ]]; then | |
| exit $TEST_EXIT_CODE | |
| fi | |
| ' | |
| - name: Generate coverage information | |
| run: | | |
| docker exec -t ${{ env.container_name }} /bin/bash -c ' | |
| source ~/.bashrc | |
| unset GREP_OPTIONS | |
| source ${{ github.workspace }}/../../../proxy | |
| source ${ci_scripts}/utils.sh; check_coverage | |
| ' | |
| - name: Upload coverage report | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| directory: build/coverage_files | |
| - name: Determine whether the coverage rate reaches 90% | |
| run: | | |
| docker exec -t ${{ env.container_name }} /bin/bash -c ' | |
| source ~/.bashrc | |
| unset GREP_OPTIONS | |
| source ${{ github.workspace }}/../../../proxy | |
| source ${ci_scripts}/utils.sh | |
| bash $ci_scripts/coverage_judge.sh | |
| COVERAGE_EXIT_CODE=$? | |
| echo $COVERAGE_EXIT_CODE | |
| if [ "$COVERAGE_EXIT_CODE" -ne 0 ]; then | |
| echo "Coverage check failed, unit tests have all passed, please do not rerun, check whether the newly added code lines are fully covered by unit tests. If you have any questions, please contact XieYunshen." | |
| exit $COVERAGE_EXIT_CODE | |
| else | |
| echo "Coverage passed" | |
| fi | |
| ' | |
| - name: Terminate and delete the container | |
| if: always() | |
| run: | | |
| set +e | |
| rm Paddle.tar.gz | |
| docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*' | |
| docker stop ${{ env.container_name }} | |
| docker rm ${{ env.container_name }} |