Skip to content

Commit 4be10d4

Browse files
YeenyeongYeenyeong
authored
Merge branch 'develop' into dev-argsort-spmd-rule
2 parents b4026b7 + bc5df86 commit 4be10d4

File tree

1,181 files changed

+65493
-10458
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,181 files changed

+65493
-10458
lines changed

‎.github/actions/check-bypass/action.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ runs:
1818
- id: check-bypass
1919
name: Check Bypass
2020
env:
21-
CI_TEAM_MEMBERS: '["SigureMo", "risemeup1", "tianshuo78520a", "0x3878f", "swgu98", "luotao1"]'
21+
CI_TEAM_MEMBERS: '["SigureMo", "risemeup1", "tianshuo78520a", "0x3878f", "swgu98", "luotao1", "XieYunshen"]'
2222
uses: PFCCLab/ci-bypass@v1
2323
with:
2424
github-token: ${{ inputs.github-token }}
@@ -29,12 +29,12 @@ runs:
2929
"any": [
3030
{
3131
"type": "labeled",
32-
"label": ["skip-ci: ${{ inputs.workflow-name }}"],
32+
"label": ["skip-ci: ${{ inputs.workflow-name }}", "skip-ci: all"],
3333
"username": ${{ env.CI_TEAM_MEMBERS }}
3434
},
3535
{
3636
"type": "commented",
37-
"comment-pattern": [".*/skip-ci ${{ inputs.workflow-name }}.*"],
37+
"comment-pattern": [".*/skip-ci ${{ inputs.workflow-name }}.*", ".*/skip-ci all.*"],
3838
"username": ${{ env.CI_TEAM_MEMBERS }}
3939
}
4040
]

‎.github/actions/rerun-workflow/rerun.sh

+6-1
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,13 @@ if [ -n "$run_ids" ]; then
4949

5050
echo "Jobs Response for run_id $run_id: $jobs_response"
5151

52-
block_jobs=$(echo "$jobs_response" | jq -r --arg job_name "$JOB_NAME" \
52+
if [[ "$JOB_NAME" == *"bypass"* ]]; then
53+
block_jobs=$(echo "$jobs_response" | jq -r --arg job_name "$JOB_NAME" \
54+
'.jobs[] | select(.name == $job_name) | .id')
55+
else
56+
block_jobs=$(echo "$jobs_response" | jq -r --arg job_name "$JOB_NAME" \
5357
'.jobs[] | select(.name == $job_name and .conclusion != "success") | .id')
58+
fi
5459

5560
if [ -n "$block_jobs" ]; then
5661
echo "Found block jobs for run_id $run_id: $block_jobs"

‎.github/workflows/Build-develop.yml

+172
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
name: Build-develop
2+
3+
on:
4+
push:
5+
branches: [develop]
6+
7+
permissions: read-all
8+
9+
defaults:
10+
run:
11+
shell: bash
12+
13+
env:
14+
COMMIT_ID: ${{ github.sha }}
15+
work_dir: /paddle
16+
PADDLE_ROOT: /paddle
17+
TASK: paddle-CI-bbild-develop
18+
ci_scripts: /paddle/ci
19+
BRANCH: ${{ github.base.ref }}
20+
CI_name: build-develop
21+
no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn"
22+
23+
jobs:
24+
build-docker:
25+
name: Develop build docker
26+
uses: ./.github/workflows/docker.yml
27+
28+
build:
29+
name: Build
30+
needs: build-docker
31+
runs-on:
32+
group: GZ_BD-CPU
33+
steps:
34+
- name: Checkout paddle
35+
uses: actions/checkout@v4
36+
with:
37+
submodules: 'recursive'
38+
fetch-depth: 1000
39+
40+
- name: Merge PR to test branch
41+
run: |
42+
git config --unset http.https://github.com/.extraheader
43+
git submodule foreach --recursive sh -c "git config --local --unset-all 'http.https://github.com/.extraheader'"
44+
git submodule foreach --recursive sh -c "git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'"
45+
bash ${ci_scripts}/third_party_tag.sh
46+
47+
- name: Check docker image and run container
48+
env:
49+
PADDLE_CUDA_INSTALL_REQUIREMENTS: "ON"
50+
FLAGS_fraction_of_gpu_memory_to_use: 0.15
51+
CTEST_OUTPUT_ON_FAILURE: 1
52+
CTEST_PARALLEL_LEVEL: 2
53+
WITH_GPU: "ON"
54+
CUDA_ARCH_NAME: Volta
55+
WITH_COVERAGE: "OFF"
56+
PADDLE_FRACTION_GPU_MEMORY_TO_USE: 0.15
57+
CUDA_VISIBLE_DEVICES: 0,1
58+
PRECISION_TEST: "ON"
59+
WITH_CINN: "ON"
60+
INFERENCE_DEMO_INSTALL_DIR: /home/data/cfs/.cache/build
61+
WITH_INCREMENTAL_COVERAGE: "OFF"
62+
WITH_AVX: "ON"
63+
WITH_TESTING: "OFF"
64+
COVERALLS_UPLOAD: "OFF"
65+
GIT_PR_ID: ${{ github.event.pull_request.number }}
66+
PADDLE_VERSION: 0.0.0
67+
WITH_DISTRIBUTE: "ON"
68+
WITH_UNITY_BUILD: "ON"
69+
PY_VERSION: "3.10"
70+
WITH_SHARED_PHI: "ON"
71+
CCACHE_MAXSIZE: 150G
72+
CCACHE_LIMIT_MULTIPLE: 0.8
73+
CCACHE_STATSLOG: /paddle/build/.stats.log
74+
CCACHE_SLOPPINESS: clang_index_store,time_macros,include_file_mtime
75+
CACHE_DIR: /home/data/cfs/.cache/build
76+
CCACHE_DIR: /home/data/cfs/.ccache/build
77+
run: |
78+
container_name=${TASK}-$(date +%Y%m%d-%H%M%S)
79+
echo "container_name=${container_name}" >> ${{ github.env }}
80+
docker_image=${{ needs.build-docker.outputs.docker_build_image }}
81+
docker pull $docker_image
82+
docker run -d -t --name ${container_name} \
83+
-v "/home/data/cfs:/home/data/cfs" \
84+
-v "/home/data/cfs/.cache:/root/.cache" \
85+
-v "/home/data/cfs/.ccache:/root/.ccache" \
86+
-v "/dev/shm:/dev/shm" \
87+
-v ${{ github.workspace }}/../../..:${{ github.workspace }}/../../.. \
88+
-v ${{ github.workspace }}:/paddle \
89+
-e BRANCH \
90+
-e COMMIT_ID \
91+
-e work_dir \
92+
-e PADDLE_ROOT \
93+
-e ci_scripts \
94+
-e PADDLE_CUDA_INSTALL_REQUIREMENTS \
95+
-e FLAGS_fraction_of_gpu_memory_to_use \
96+
-e CTEST_OUTPUT_ON_FAILURE \
97+
-e CTEST_PARALLEL_LEVEL \
98+
-e WITH_GPU \
99+
-e CUDA_ARCH_NAME \
100+
-e WITH_COVERAGE \
101+
-e PADDLE_FRACTION_GPU_MEMORY_TO_USE \
102+
-e CUDA_VISIBLE_DEVICES \
103+
-e PRECISION_TEST \
104+
-e WITH_CINN \
105+
-e INFERENCE_DEMO_INSTALL_DIR \
106+
-e WITH_INCREMENTAL_COVERAGE \
107+
-e WITH_SHARED_PHI \
108+
-e WITH_TESTING \
109+
-e COVERALLS_UPLOAD \
110+
-e GIT_PR_ID \
111+
-e PADDLE_VERSION \
112+
-e WITH_DISTRIBUTE \
113+
-e WITH_UNITY_BUILD \
114+
-e PY_VERSION \
115+
-e CCACHE_MAXSIZE \
116+
-e CCACHE_LIMIT_MULTIPLE \
117+
-e WITH_AVX \
118+
-e CACHE_DIR \
119+
-e CCACHE_DIR \
120+
-e CCACHE_SLOPPINESS \
121+
-e CCACHE_STATSLOG \
122+
-e no_proxy \
123+
-e CI_name \
124+
-w /paddle --network host ${docker_image}
125+
126+
- name: Set git config
127+
env:
128+
work_dir: ${{ github.workspace }}
129+
run: |
130+
docker exec -t ${{ env.container_name }} /bin/bash -c '
131+
git config --global --add safe.directory ${work_dir}
132+
git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path"
133+
git config --global user.name "PaddleCI"
134+
git config --global user.email "paddle_ci@example.com"
135+
git remote add upstream https://github.com/PaddlePaddle/Paddle.git
136+
'
137+
138+
- name: Build
139+
run: |
140+
docker exec -t ${{ env.container_name }} /bin/bash -c '
141+
mkdir -p /root/.cache/build
142+
mkdir -p /root/.ccache/build
143+
source ${{ github.workspace }}/../../../proxy
144+
bash ${ci_scripts}/run_setup.sh bdist_wheel
145+
'
146+
147+
- name: Upload paddle_whl
148+
env:
149+
home_path: ${{ github.workspace }}/..
150+
bos_file: ${{ github.workspace }}/../bos/BosClient.py
151+
paddle_whl: paddlepaddle_gpu-0.0.0-cp310-cp310-linux_x86_64.whl
152+
run: |
153+
docker exec -t ${{ env.container_name }} /bin/bash -c '
154+
export AK=paddle
155+
export SK=paddle
156+
if [ ! -f "${{ env.bos_file }}" ]; then
157+
wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate
158+
mkdir ${{ env.home_path }}/bos
159+
tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos
160+
fi
161+
mv /paddle/dist/${{ env.paddle_whl }} .
162+
echo "Uploading paddle_whl to bos"
163+
python ${{ env.bos_file }} ${{ env.paddle_whl }} paddle-github-action/PR/build/0/${{ env.COMMIT_ID }}
164+
'
165+
166+
- name: Terminate and delete the container
167+
if: always()
168+
run: |
169+
set +e
170+
docker exec -t ${{ env.container_name }} /bin/bash -c 'rm -rf * .[^.]*'
171+
docker stop ${{ env.container_name }}
172+
docker rm ${{ env.container_name }}

‎.github/workflows/CI.yml

+41-6
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ name: CI
33
on:
44
pull_request:
55
types: [opened, synchronize]
6+
branches: [develop, release/**]
67

78
permissions: read-all
89

@@ -19,21 +20,55 @@ jobs:
1920
name: Clone-linux
2021
uses: ./.github/workflows/_Clone-linux.yml
2122

23+
build-docker:
24+
name: build docker images
25+
needs: clone
26+
uses: ./.github/workflows/docker.yml
27+
2228
sot:
2329
name: PR-CI-SOT
2430
uses: ./.github/workflows/_SOT.yml
31+
needs: build-docker
32+
with:
33+
docker_cpu_image: ${{ needs.build-docker.outputs.docker_cpu_image }}
34+
35+
xpu:
36+
name: Linux-XPU
37+
uses: ./.github/workflows/_Linux-XPU.yml
2538
needs: clone
2639

2740
inference:
2841
name: PR-CI-Inference
2942
uses: ./.github/workflows/_Inference.yml
43+
needs: build-docker
44+
with:
45+
docker_inference_image: ${{ needs.build-docker.outputs.docker_inference_image }}
46+
47+
cpu:
48+
name: Linux-CPU
49+
uses: ./.github/workflows/_Linux-CPU.yml
3050
needs: clone
3151

32-
coverage:
33-
name: PR-CI-Coverage
34-
uses: ./.github/workflows/_Coverage.yml
52+
npu:
53+
name: Linux-NPU
54+
uses: ./.github/workflows/_Linux-NPU.yml
55+
needs: cpu
56+
57+
distribute:
58+
name: Distribute-stable
59+
uses: ./.github/workflows/_Distribute-stable.yml
3560
needs: clone
3661

37-
# Py3:
38-
# uses: ./.github/workflows/_Py3.yml
39-
# needs: Clone
62+
build:
63+
name: Linux-build
64+
uses: ./.github/workflows/_Linux-build.yml
65+
needs: build-docker
66+
with:
67+
docker_build_image: ${{ needs.build-docker.outputs.docker_build_image }}
68+
69+
static-check:
70+
name: Static-Check
71+
uses: ./.github/workflows/_Static-Check.yml
72+
needs: build
73+
with:
74+
docker_build_image: ${{ needs.build.outputs.docker_build_image }}

0 commit comments

Comments
 (0)