diff --git a/.github/workflows/CI-test.yml b/.github/workflows/CI-test.yml new file mode 100644 index 00000000000000..0d4a0d41492ab5 --- /dev/null +++ b/.github/workflows/CI-test.yml @@ -0,0 +1,51 @@ +name: CI-test + +on: + pull_request: + types: [opened, synchronize] + +permissions: read-all + +concurrency: + group: ${{ github.event.pull_request.number }}-test + cancel-in-progress: true + +env: + PR_ID: ${{ github.event.pull_request.number }} + COMMIT_ID: ${{ github.event.pull_request.head.sha }} + +jobs: + clone: + name: Clone-linux-test + uses: ./.github/workflows/_Clone-linux.yml + with: + pr_dir: PR-test + + + xpu: + name: Linux-XPU-test + uses: ./.github/workflows/_Linux-XPU.yml + with: + pr_dir: PR-test + needs: clone + + coverage: + name: PR-CI-Coverage-test + uses: ./.github/workflows/_Coverage.yml + with: + pr_dir: PR-test + needs: clone + + cpu: + name: Linux-CPU-test + uses: ./.github/workflows/_CPU.yml + with: + pr_dir: PR-test + needs: clone + + npu: + name: Linux-NPU-test + uses: ./.github/workflows/_NPU.yml + with: + pr_dir: PR-test + needs: cpu diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 88000a779c2dcf..381a0dd2828c79 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -18,33 +18,19 @@ jobs: clone: name: Clone-linux uses: ./.github/workflows/_Clone-linux.yml + with: + pr_dir: PR sot: name: PR-CI-SOT uses: ./.github/workflows/_SOT.yml - needs: clone - - xpu: - name: Linux-XPU - uses: ./.github/workflows/_Linux-XPU.yml + with: + pr_dir: PR needs: clone inference: name: PR-CI-Inference uses: ./.github/workflows/_Inference.yml + with: + pr_dir: PR needs: clone - - coverage: - name: PR-CI-Coverage - uses: ./.github/workflows/_Coverage.yml - needs: clone - - cpu: - name: Linux-CPU - uses: ./.github/workflows/_CPU.yml - needs: clone - - npu: - name: Linux-NPU - uses: ./.github/workflows/_NPU.yml - needs: cpu diff --git a/.github/workflows/_CPU.yml b/.github/workflows/_CPU.yml index 4ce63c77c433da..8ff237b5297d5d 100644 --- a/.github/workflows/_CPU.yml +++ b/.github/workflows/_CPU.yml @@ -2,6 +2,10 @@ name: PR-CI-CPU on: workflow_call: + inputs: + pr_dir: + required: true + type: string env: dockerfile: Dockerfile.cuda9_cudnn7_gcc48_py35_centos6 @@ -91,7 +95,7 @@ jobs: rm -rf * .[^.]* set -e echo "Downloading Paddle.tar.gz" - wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate + wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/${{ inputs.pr_dir }}/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate echo "Extracting Paddle.tar.gz" tar xf Paddle.tar.gz --strip-components=1 rm Paddle.tar.gz @@ -151,7 +155,7 @@ jobs: fi cd dist echo "Uploading paddle_whl to bos" - python3.9 ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/cpu_whl/${{ env.PR_ID }}/${{ env.COMMIT_ID }} + python3.9 ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/${{ inputs.pr_dir }}/cpu_whl/${{ env.PR_ID }}/${{ env.COMMIT_ID }} ' - name: Terminate and delete the container diff --git a/.github/workflows/_Clone-linux.yml b/.github/workflows/_Clone-linux.yml index e095dafd4eb1c0..94049be8d3bf85 100644 --- a/.github/workflows/_Clone-linux.yml +++ b/.github/workflows/_Clone-linux.yml @@ -2,6 +2,10 @@ name: Clone-linux on: workflow_call: + inputs: + pr_dir: + required: true + type: string permissions: read-all @@ -70,7 +74,7 @@ jobs: echo "::group::Install bce-python-sdk" python -m pip install bce-python-sdk==0.8.74 echo "::endgroup::" - python ${bos_file} Paddle.tar.gz paddle-github-action/PR/Paddle/${PR_ID}/${COMMIT_ID} + python ${bos_file} Paddle.tar.gz paddle-github-action/${{ inputs.pr_dir }}/Paddle/${PR_ID}/${COMMIT_ID} rm Paddle.tar.gz cd - git switch ${{ github.event.pull_request.base.ref }} diff --git a/.github/workflows/_Coverage.yml b/.github/workflows/_Coverage.yml index 9d8c47fc1aac48..a8856c9f0d69cd 100644 --- a/.github/workflows/_Coverage.yml +++ b/.github/workflows/_Coverage.yml @@ -2,6 +2,10 @@ name: PR-CI-Coverage on: workflow_call: + inputs: + pr_dir: + required: true + type: string env: Dockerfile: Dockerfile.cuda117_cudnn8_gcc82_ubuntu18_coverage @@ -33,7 +37,7 @@ jobs: run: | set -e echo "Downloading Paddle.tar.gz" - wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate + wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/${{ inputs.pr_dir }}/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate echo "Extracting Paddle.tar.gz" tar xf Paddle.tar.gz --strip-components=1 rm Paddle.tar.gz @@ -195,12 +199,12 @@ jobs: fi cd /paddle/dist echo "Uploading coverage build size" - python ${{ env.bos_file }} coverage_build_size paddle-github-action/PR/coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }} + python ${{ env.bos_file }} coverage_build_size paddle-github-action/${{ inputs.pr_dir }}/coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }} echo "Uploading coverage wheel" - python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }} + python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/${{ inputs.pr_dir }}/coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }} cd / echo "Uploading Paddle.tar.gz" - python ${{ env.bos_file }} Paddle.tar.gz paddle-github-action/PR/coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }} + python ${{ env.bos_file }} Paddle.tar.gz paddle-github-action/${{ inputs.pr_dir }}/coverage/${{ env.PR_ID }}/${{ env.COMMIT_ID }} rm Paddle.tar.gz ' @@ -224,7 +228,7 @@ jobs: run: | set -e echo "Downloading Paddle.tar.gz" - wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/coverage/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate + wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/${{ inputs.pr_dir }}/coverage/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate echo "Extracting Paddle.tar.gz" tar --use-compress-program='pzstd -1' -xpf Paddle.tar.gz --strip-components=1 rm Paddle.tar.gz diff --git a/.github/workflows/_Inference.yml b/.github/workflows/_Inference.yml index bfa766ea98b5f4..77fe211670da21 100644 --- a/.github/workflows/_Inference.yml +++ b/.github/workflows/_Inference.yml @@ -2,6 +2,10 @@ name: PR-CI-Inference on: workflow_call: + inputs: + pr_dir: + required: true + type: string env: dockerfile: Dockerfile.cuda11.2_cudnn8.1_trt8.4_gcc8.2_ubuntu18 @@ -107,7 +111,7 @@ jobs: docker exec -t ${container_name} /bin/bash -c ' set -e echo "Downloading Paddle.tar.gz" - wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate + wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/${{ inputs.pr_dir }}/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate echo "Extracting Paddle.tar.gz" tar xf Paddle.tar.gz --strip-components=1 rm Paddle.tar.gz @@ -151,7 +155,7 @@ jobs: source /action-runner/unproxy cd / tar --use-compress-program="pigz -1" -cpPf build.tar.gz paddle - python ${{ env.bos_file }} build.tar.gz paddle-github-action/PR/inference/${{ env.PR_ID }}/${{ env.COMMIT_ID }} + python ${{ env.bos_file }} build.tar.gz paddle-github-action/${{ inputs.pr_dir }}/inference/${{ env.PR_ID }}/${{ env.COMMIT_ID }} rm build.tar.gz ' @@ -244,7 +248,7 @@ jobs: docker exec -t ${{ env.container_name }} /bin/bash -c ' set -e echo "Downloading build.tar.gz" - wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/inference/${PR_ID}/${COMMIT_ID}/build.tar.gz --no-check-certificate + wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/${{ inputs.pr_dir }}/inference/${PR_ID}/${COMMIT_ID}/build.tar.gz --no-check-certificate echo "Extracting build.tar.gz" tar --use-compress-program="pigz -1" -xpf build.tar.gz --strip-components=1 rm build.tar.gz diff --git a/.github/workflows/_Linux-XPU.yml b/.github/workflows/_Linux-XPU.yml index eacba9e1d1cb88..0fefb9d9af14a4 100644 --- a/.github/workflows/_Linux-XPU.yml +++ b/.github/workflows/_Linux-XPU.yml @@ -2,6 +2,10 @@ name: Linux-XPU on: workflow_call: + inputs: + pr_dir: + required: true + type: string env: dockerfile: dockerfile @@ -42,7 +46,7 @@ jobs: run: | set -e echo "Downloading Paddle.tar.gz" - wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate + wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/${{ inputs.pr_dir }}/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate echo "Extracting Paddle.tar.gz" tar xf Paddle.tar.gz --strip-components=1 rm Paddle.tar.gz @@ -146,10 +150,10 @@ jobs: cd .. tar --use-compress-program="pigz" -cpf build.tar.gz Paddle # source /home/opt/deck/1.0/etc/bashrc - python3 ${bos_file} build.tar.gz paddle-github-action/PR/xpu/${{ env.PR_ID }}/${{ env.COMMIT_ID }} + python3 ${bos_file} build.tar.gz paddle-github-action/${{ inputs.pr_dir }}/xpu/${{ env.PR_ID }}/${{ env.COMMIT_ID }} rm build.tar.gz cp ${{ github.workspace }}/dist/$paddle_whl . - python3 ${bos_file} ${paddle_whl} paddle-github-action/PR/xpu/${{ env.PR_ID }}/${{ env.COMMIT_ID }} + python3 ${bos_file} ${paddle_whl} paddle-github-action/${{ inputs.pr_dir }}/xpu/${{ env.PR_ID }}/${{ env.COMMIT_ID }} rm ${paddle_whl} - name: Terminate and delete the container @@ -172,7 +176,7 @@ jobs: - name: Download build.tar.gz run: | sudo rm -rf * .[^.]* - wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/xpu/${PR_ID}/${COMMIT_ID}/build.tar.gz --no-check-certificate + wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/${{ inputs.pr_dir }}/xpu/${PR_ID}/${COMMIT_ID}/build.tar.gz --no-check-certificate # wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/xpu/72068/a58543d2f92bbe5c817e185e9cc24b549c919601/build.tar.gz --no-check-certificate tar --use-compress-program="pigz" -xpf build.tar.gz --strip-components=1 rm build.tar.gz diff --git a/.github/workflows/_NPU.yml b/.github/workflows/_NPU.yml index 2ecafbde677860..03b477e9006459 100644 --- a/.github/workflows/_NPU.yml +++ b/.github/workflows/_NPU.yml @@ -2,6 +2,10 @@ name: PR-CI-NPU-910B-Paddle on: workflow_call: + inputs: + pr_dir: + required: true + type: string env: dockerfile: dockerfile @@ -39,7 +43,7 @@ jobs: source ~/.bashrc set -e echo "Downloading Paddle.tar.gz" - wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate + wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/${{ inputs.pr_dir }}/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate echo "Extracting Paddle.tar.gz" tar xf Paddle.tar.gz rm Paddle.tar.gz @@ -141,7 +145,7 @@ jobs: source ~/.bashrc wget -q https://sys-p0.bj.bcebos.com/libstdc%2B%2B6_13.1.0-8ubuntu1_20.04.2_amd64.deb dpkg -i *deb - wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/cpu_whl/${PR_ID}/${COMMIT_ID}/paddlepaddle-0.0.0-cp39-cp39-linux_x86_64.whl --no-check-certificate + wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/${{ inputs.pr_dir }}/cpu_whl/${PR_ID}/${COMMIT_ID}/paddlepaddle-0.0.0-cp39-cp39-linux_x86_64.whl --no-check-certificate PATH=/usr/local/bin:${PATH} echo "export PATH=$PATH" >> ~/.bashrc ln -sf $(which python3.9) /usr/local/bin/python diff --git a/.github/workflows/_SOT.yml b/.github/workflows/_SOT.yml index 50c6587e870c11..3def3af0436366 100644 --- a/.github/workflows/_SOT.yml +++ b/.github/workflows/_SOT.yml @@ -2,6 +2,10 @@ name: PR-CI-SOT on: workflow_call: + inputs: + pr_dir: + required: true + type: string env: dockerfile: Dockerfile.cuda9_cudnn7_gcc48_py35_centos6 @@ -99,7 +103,7 @@ jobs: docker exec -t ${{ env.container_name }} /bin/bash -c ' rm -rf * .[^.]* set -e - wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate + wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/${{ inputs.pr_dir }}/Paddle/${PR_ID}/${COMMIT_ID}/Paddle.tar.gz --no-check-certificate tar xf Paddle.tar.gz --strip-components=1 rm Paddle.tar.gz git config --global user.name "PaddleCI"