Merge branch 'PaddlePaddle:develop' into trt_debug_info

Author: PolaKuma

.github/actions/check-bypass/action.yml

@@ -18,7 +18,7 @@ runs:
     - id: check-bypass
       name: Check Bypass
       env:
-        CI_TEAM_MEMBERS: '["SigureMo", "risemeup1", "tianshuo78520a", "0x3878f", "swgu98", "luotao1"]'
+        CI_TEAM_MEMBERS: '["SigureMo", "risemeup1", "tianshuo78520a", "0x3878f", "swgu98", "luotao1", "XieYunshen"]'
       uses: PFCCLab/ci-bypass@v1
       with:
         github-token: ${{ inputs.github-token }}
@@ -29,12 +29,12 @@ runs:
             "any": [
               {
                 "type": "labeled",
-                "label": ["skip-ci: ${{ inputs.workflow-name }}"],
+                "label": ["skip-ci: ${{ inputs.workflow-name }}", "skip-ci: all"],
                 "username": ${{ env.CI_TEAM_MEMBERS }}
               },
               {
                 "type": "commented",
-                "comment-pattern": [".*/skip-ci ${{ inputs.workflow-name }}.*"],
+                "comment-pattern": [".*/skip-ci ${{ inputs.workflow-name }}.*", ".*/skip-ci all.*"],
                 "username": ${{ env.CI_TEAM_MEMBERS }}
               }
             ]

.github/workflows/CI.yml

@@ -3,6 +3,7 @@ name: CI
 on:
   pull_request:
     types: [opened, synchronize]
+    branches: [develop, release/**]
 
 permissions: read-all
 
@@ -24,16 +25,27 @@ jobs:
     uses: ./.github/workflows/_SOT.yml
     needs: clone
 
+  xpu:
+    name: Linux-XPU
+    uses: ./.github/workflows/_Linux-XPU.yml
+    needs: clone
+
   inference:
     name: PR-CI-Inference
     uses: ./.github/workflows/_Inference.yml
     needs: clone
 
   coverage:
-    name: PR-CI-Coverage
+    name: Coverage
     uses: ./.github/workflows/_Coverage.yml
     needs: clone
 
-  # Py3:
-  #   uses: ./.github/workflows/_Py3.yml
-  #   needs: Clone
+  cpu:
+    name: Linux-CPU
+    uses: ./.github/workflows/_Linux-CPU.yml
+    needs: clone
+
+  npu:
+    name: Linux-NPU
+    uses: ./.github/workflows/_Linux-NPU.yml
+    needs: cpu

.github/workflows/_Coverage.yml

@@ -1,4 +1,4 @@
-name: PR-CI-Coverage
+name: Coverage
 
 on:
   workflow_call:
@@ -171,7 +171,7 @@ jobs:
           rm -rf ./python/build/lib.linux-x86_64-3.9/
           cd "${work_dir}/build/third_party" && find $(ls | grep -v "dlpack" | grep -v "install" | grep -v "eigen3" | grep -v "gflags") -type f ! -name "*.so" -a ! -name "libdnnl.so*" -delete
           cd /
-          tar --use-compress-program="pzstd -1" -cpf Paddle.tar.gz paddle
+          tar --use-compress-program="pzstd -1" -cf Paddle.tar.gz paddle
           '
 
       - name: Upload coverage product
@@ -218,21 +218,8 @@ jobs:
     needs: build
     if: needs.build.outputs.can-skip != 'true'
     runs-on:
-      group: BD_BJ_v100
+      group: BD_BJ-V100
     steps:
-      - name: Download paddle.tar.gz and update test branch
-        run: |
-          set -e
-          echo "Downloading Paddle.tar.gz"
-          wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/coverage/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
-          echo "Extracting Paddle.tar.gz"
-          tar  --use-compress-program='pzstd -1' -xpf Paddle.tar.gz --strip-components=1
-          rm Paddle.tar.gz
-          source ${{ github.workspace }}/../../../proxy
-          git checkout test
-          echo "Pull upstream develop"
-          git pull upstream $BRANCH --no-edit
-
       - name: Check docker image and run container
         env:
           CACHE_DIR: "/root/.cache/coverage"
@@ -303,6 +290,22 @@ jobs:
             -e GITHUB_API_TOKEN  \
             -w /paddle --network host ${docker_image}
 
+      - name: Download paddle.tar.gz and update test branch
+        run: |
+          docker exec -t ${{ env.container_name }} /bin/bash -c '
+          rm -rf * .[^.]*
+          set -e
+          echo "Downloading Paddle.tar.gz"
+          wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/coverage/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
+          echo "Extracting Paddle.tar.gz"
+          tar --use-compress-program="pzstd -1" -xf Paddle.tar.gz --strip-components=1
+          rm Paddle.tar.gz
+          source ${{ github.workspace }}/../../../proxy
+          git checkout test
+          echo "Pull upstream develop"
+          git pull upstream $BRANCH --no-edit
+          '
+
       - name: Test
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
@@ -315,33 +318,43 @@ jobs:
           fi
           '
 
-      - name: Check coverage data
+      - name: Generate coverage information
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
           source ~/.bashrc
           unset GREP_OPTIONS
           source ${{ github.workspace }}/../../../proxy
           source ${ci_scripts}/utils.sh; check_coverage
-          COVERAGE_EXIT_CODE=$?
-          echo "COVERAGE_EXIT_CODE=${COVERAGE_EXIT_CODE}" >> ${{ github.env }}
           '
 
       - name: Upload coverage report
         uses: codecov/codecov-action@v4
         with:
           directory: build/coverage_files
 
-      - name: Judge coverage passed or not
+      - name: Determine whether the coverage rate reaches 90%
         run: |
-          if [ "${{ env.COVERAGE_EXIT_CODE }}" -ne 0 ]; then
-            echo "Coverage failed"
-            exit ${{ env.COVERAGE_EXIT_CODE }}
+          docker exec -t ${{ env.container_name }} /bin/bash -c '
+          source ~/.bashrc
+          unset GREP_OPTIONS
+          source ${{ github.workspace }}/../../../proxy
+          source ${ci_scripts}/utils.sh
+          bash $ci_scripts/coverage_judge.sh
+          COVERAGE_EXIT_CODE=$?
+          echo $COVERAGE_EXIT_CODE
+          if [ "$COVERAGE_EXIT_CODE" -ne 0 ]; then
+            echo "Coverage check failed, unit tests have all passed, please do not rerun, check whether the newly added code lines are fully covered by unit tests. If you have any questions, please contact XieYunshen."
+            exit $COVERAGE_EXIT_CODE
+          else
+            echo "Coverage passed"
           fi
+          '
 
       - name: Terminate and delete the container
         if: always()
         run: |
           set +e
+          rm Paddle.tar.gz
           docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
           docker stop ${{ env.container_name }}
           docker rm ${{ env.container_name }}

.github/workflows/_Py3.yml renamed to .github/workflows/_Linux-CPU.yml

@@ -1,31 +1,33 @@
-name: PR-CI-Py3
+name: Linux-CPU
 
 on:
   workflow_call:
 
 env:
   dockerfile: Dockerfile.cuda9_cudnn7_gcc48_py35_centos6
-  docker_image: 077ca344c322
+  docker_image: 3d2fa88da0da
   PR_ID: ${{ github.event.pull_request.number }}
   COMMIT_ID: ${{ github.event.pull_request.head.sha }}
-  TASK: paddle-CI-${{ github.event.pull_request.number }}-py3
+  work_dir: /paddle
+  PADDLE_ROOT: /paddle
+  TASK: paddle-CI-${{ github.event.pull_request.number }}-cpu
   ci_scripts: ${{ github.workspace }}/ci
+  BRANCH: ${{ github.event.pull_request.base.ref }}
+  CI_name: cpu
 
 defaults:
   run:
     shell: bash
 
 jobs:
   build-and-test:
+    name: Build and test
     runs-on:
       group: GZ_BD-CPU
 
     steps:
       - name: Check docker image  and run container
         env:
-          BRANCH: develop
-          work_dir: ${{ github.workspace }}
-          PADDLE_ROOT: ${{ github.workspace }}
           WITH_SHARED_PHI: "ON"
           WITH_MKL: "OFF"
           WITH_TESTING: "ON"
@@ -43,17 +45,17 @@ jobs:
           CCACHE_MAXSIZE: 150G
           CCACHE_LIMIT_MULTIPLE: 0.8
           WITH_AVX: "OFF"
-          CCACHE_DIR: "/root/.ccache/py3"
+          CCACHE_DIR: "/root/.ccache/cpu"
         run: |
-          # bash ${{ github.workspace }}/paddle/scripts/paddle_build/check_docker_image.sh
-          container_name=${TASK}-$(date +%s)
+          container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
           echo "container_name=${container_name}" >> ${{ github.env }}
           docker run -d -t --name ${container_name} \
             -v "/home/data/cfs:/home/data/cfs" \
-            -v "/home/data/cfs/.cache:/root/.cache" \
+            -v "/home/data/cfs/.cache/python35-cpu:/root/.cache" \
             -v "/home/data/cfs/.ccache:/root/.ccache" \
             -v "/dev/shm:/dev/shm"  \
-            -v "${{ github.workspace }}/../../../..:${{ github.workspace }}/../../../.." \
+            -v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
+            -v ${{ github.workspace }}:/paddle \
             -e BRANCH \
             -e PR_ID \
             -e COMMIT_ID \
@@ -79,49 +81,54 @@ jobs:
             -e ci_scripts \
             -e WITH_AVX \
             -e CCACHE_DIR \
-            -w ${{ github.workspace }} --network host ${docker_image}
+            -w /paddle --network host ${docker_image}
 
-      - name: Download paddle.tar.gz and merge develop
+      - name: Download paddle.tar.gz and merge target branch
         env:
           work_dir: ${{ github.workspace }}
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
-          rm -rf * .[^.]*
-          rm -rf /root/.cache && ln -s /home/data/cfs/.cache/python35-cpu /root/.cache
+          rm -rf  * .[^.]*
           set -e
+          echo "Downloading Paddle.tar.gz"
           wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate
+          echo "Extracting Paddle.tar.gz"
           tar xf Paddle.tar.gz --strip-components=1
           rm Paddle.tar.gz
           git config --global user.name "PaddleCI"
           git config --global user.email "paddle_ci@example.com"
           git remote add upstream https://github.com/PaddlePaddle/Paddle.git
-          . ../../../proxy
-          git checkout develop
-          git pull upstream develop
-          git fetch upstream develop
+          source ${{ github.workspace }}/../../../proxy
+          git checkout $BRANCH
+          git pull upstream $BRANCH
+          git fetch upstream $BRANCH
           git checkout test
-          git merge --no-edit develop
+          git merge --no-edit $BRANCH
           '
 
+      - name: Check bypass
+        id: check-bypass
+        uses: ./.github/actions/check-bypass
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          workflow-name: cpu
+
       - name: Build
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
-          . ../../../proxy
-          bash ${ci_scripts}/run_setup.sh ${PYTHON_ABI:-""} bdist_wheel ${parallel_number:-""} py3
+          source ${{ github.workspace }}/../../../proxy
+          bash ${ci_scripts}/run_setup.sh bdist_wheel
           EXCODE=$?
           exit $EXCODE
           '
 
       - name: Test
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
-          . ../../../proxy
+          source ${{ github.workspace }}/../../../proxy
           bash ${ci_scripts}/run_linux_cpu_test.sh
           EXCODE=$?
           source ${ci_scripts}/utils.sh; clean_build_files
-          echo "::group::Install bce-python-sdk"
-          python -m pip install bce-python-sdk==0.8.74
-          echo "::endgroup::"
           exit $EXCODE
           '
 
@@ -132,14 +139,20 @@ jobs:
           paddle_whl: paddlepaddle-0.0.0-cp39-cp39-linux_x86_64.whl
         run: |
           docker exec -t ${{ env.container_name }} /bin/bash -c '
+          echo "::group::Install bce-python-sdk"
+          source ${{ github.workspace }}/../../../proxy
+          python -m pip install bce-python-sdk==0.8.74
+          echo "::endgroup::"
           export AK=paddle
           export SK=paddle
           if [ ! -f "${{ env.bos_file }}" ]; then
             wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
             mkdir ${{ env.home_path }}/bos
             tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
           fi
-          cd ${{ github.workspace }}/dist
+          cd dist
+          echo "Uploading paddle_whl to bos"
+          source ${{ github.workspace }}/../../../unproxy
           python3.9 ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/cpu_whl/${{ env.PR_ID }}/${{ env.COMMIT_ID }}
           '