From cfd7c381e971c6e9dfa6f882770e9fbca3633e45 Mon Sep 17 00:00:00 2001 From: gushiwei Date: Thu, 24 Apr 2025 03:02:26 +0000 Subject: [PATCH 1/4] test=document_fix tp cache --- .github/workflows/_Linux-XPU.yml | 17 +++++++++++++--- ci/download_tp_cache.sh | 33 ++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 ci/download_tp_cache.sh diff --git a/.github/workflows/_Linux-XPU.yml b/.github/workflows/_Linux-XPU.yml index eacba9e1d1cb88..cbebdcfbd8ab16 100644 --- a/.github/workflows/_Linux-XPU.yml +++ b/.github/workflows/_Linux-XPU.yml @@ -55,6 +55,12 @@ jobs: echo "Pull upstream develop or target branch" git pull upstream $BRANCH --no-edit + - name: Search cached third_party + env: + cache_dir: ${{ github.workspace }}/../../../../.cache + run: | + bash -x ${ci_scripts_runner}/download_tp_cache.sh ${cache_dir}/XPU/third_party XPU + - name: Check docker image and run container env: WITH_SHARED_PHI: "ON" @@ -130,7 +136,7 @@ jobs: exit $EXCODE ' - - name: Upload build.tar.gz and paddle_whl to bos + - name: Upload product to bos env: AK: paddle SK: paddle @@ -143,7 +149,13 @@ jobs: mkdir ${{ env.home_path }}/bos tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos fi - cd .. + if [ "${update_cached_package}" == ON ];then + cd ${{ github.workspace }} + tar --use-compress-program=pigz -cf ${xz_file} -C build third_party + fi + rm -rf `find . -name "*.a"` + rm -rf `find . -name "*.o"` + cd ${{ github.workspace }}/.. tar --use-compress-program="pigz" -cpf build.tar.gz Paddle # source /home/opt/deck/1.0/etc/bashrc python3 ${bos_file} build.tar.gz paddle-github-action/PR/xpu/${{ env.PR_ID }}/${{ env.COMMIT_ID }} @@ -173,7 +185,6 @@ jobs: run: | sudo rm -rf * .[^.]* wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/xpu/${PR_ID}/${COMMIT_ID}/build.tar.gz --no-check-certificate - # wget -q --no-proxy https://paddle-github-action.bj.bcebos.com/PR/xpu/72068/a58543d2f92bbe5c817e185e9cc24b549c919601/build.tar.gz --no-check-certificate tar --use-compress-program="pigz" -xpf build.tar.gz --strip-components=1 rm build.tar.gz diff --git a/ci/download_tp_cache.sh b/ci/download_tp_cache.sh new file mode 100644 index 00000000000000..ab72ddcfeaa82a --- /dev/null +++ b/ci/download_tp_cache.sh @@ -0,0 +1,33 @@ +# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_md5=`find ${work_dir} -path './cmake/*' -type f |xargs md5sum|md5sum |awk '{print $1}'` +third_party_md5=`find ${work_dir} -path './third_party/*' -type f |xargs md5sum|md5sum|awk '{print $1}'` +export md5_content=`echo ${cmake_md5} ${third_party_md5} |md5sum|awk '{print $1}'` +xz_dir="$1" && echo "xz_dir=${xz_dir}" >> $GITHUB_ENV +xz_file_tar="${xz_dir}/${md5_content}.tar" +xz_file="${xz_dir}/${md5_content}.tar.xz" && echo "xz_file=${xz_file}" >> $GITHUB_ENV +if [ ! -f "${xz_file}" ];then + set +e + wget -q --no-proxy --no-check-certificate https://paddle-github-action.bj.bcebos.com/home/.cache/$2/third_party/${md5_content}.tar.xz; result=$? + if [ $result -eq 0 ];then + mkdir -p ${work_dir}/build + mv ${md5_content}.tar.xz ${xz_dir} + else + mkdir -p ${xz_dir} + echo "update_cached_package=ON" >> $GITHUB_ENV + fi +else + mkdir -p ${work_dir}/build +fi From 75437eaecc73330377ea190b26725dc102c21dcf Mon Sep 17 00:00:00 2001 From: gushiwei Date: Thu, 24 Apr 2025 14:26:49 +0000 Subject: [PATCH 2/4] test=document_fix --- .github/workflows/_Linux-XPU.yml | 14 ++++++-------- ci/download_tp_cache.sh | 8 ++++---- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/_Linux-XPU.yml b/.github/workflows/_Linux-XPU.yml index cbebdcfbd8ab16..c59e0eaf332b54 100644 --- a/.github/workflows/_Linux-XPU.yml +++ b/.github/workflows/_Linux-XPU.yml @@ -57,7 +57,7 @@ jobs: - name: Search cached third_party env: - cache_dir: ${{ github.workspace }}/../../../../.cache + cache_dir: ${{ github.workspace }}/../../../../../.cache run: | bash -x ${ci_scripts_runner}/download_tp_cache.sh ${cache_dir}/XPU/third_party XPU @@ -83,7 +83,7 @@ jobs: CCACHE_LIMIT_MULTIPLE: 0.8 no_proxy: "bcebos.com,apiin.im.baidu.com,gitee.com,aliyun.com,.baidu.com,.tuna.tsinghua.edu.cn" GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} - home_dir: ${{ github.workspace }}/../../../.. + home_dir: ${{ github.workspace }}/../../../../.. run: | container_name=${TASK}-$(date +%Y%m%d-%H%M%S) echo "container_name=${container_name}" >> ${{ github.env }} @@ -132,8 +132,6 @@ jobs: git config --global --add safe.directory ${work_dir} git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path" bash -x ${ci_scripts}/run_setup.sh bdist_wheel - EXCODE=$? - exit $EXCODE ' - name: Upload product to bos @@ -144,15 +142,15 @@ jobs: bos_file: ${{ github.workspace }}/../bos/BosClient.py paddle_whl: paddlepaddle_xpu-0.0.0-cp310-cp310-linux_x86_64.whl run: | + if [ "${update_cached_package}" == ON ];then + cd ${{ github.workspace }} + tar --use-compress-program=pigz -cf ${xz_file} -C build third_party + fi if [ ! -f "${{ env.bos_file }}" ]; then wget -q --no-proxy -O ${{ env.home_path }}/bos_new.tar.gz https://xly-devops.bj.bcebos.com/home/bos_new.tar.gz --no-check-certificate mkdir ${{ env.home_path }}/bos tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos fi - if [ "${update_cached_package}" == ON ];then - cd ${{ github.workspace }} - tar --use-compress-program=pigz -cf ${xz_file} -C build third_party - fi rm -rf `find . -name "*.a"` rm -rf `find . -name "*.o"` cd ${{ github.workspace }}/.. diff --git a/ci/download_tp_cache.sh b/ci/download_tp_cache.sh index ab72ddcfeaa82a..299098fb585a2d 100644 --- a/ci/download_tp_cache.sh +++ b/ci/download_tp_cache.sh @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -cmake_md5=`find ${work_dir} -path './cmake/*' -type f |xargs md5sum|md5sum |awk '{print $1}'` -third_party_md5=`find ${work_dir} -path './third_party/*' -type f |xargs md5sum|md5sum|awk '{print $1}'` +cmake_md5=`find ${GITHUB_WORKSPACE} -path './cmake/*' -type f |xargs md5sum|md5sum |awk '{print $1}'` +third_party_md5=`find ${GITHUB_WORKSPACE} -path './third_party/*' -type f |xargs md5sum|md5sum|awk '{print $1}'` export md5_content=`echo ${cmake_md5} ${third_party_md5} |md5sum|awk '{print $1}'` xz_dir="$1" && echo "xz_dir=${xz_dir}" >> $GITHUB_ENV xz_file_tar="${xz_dir}/${md5_content}.tar" @@ -22,12 +22,12 @@ if [ ! -f "${xz_file}" ];then set +e wget -q --no-proxy --no-check-certificate https://paddle-github-action.bj.bcebos.com/home/.cache/$2/third_party/${md5_content}.tar.xz; result=$? if [ $result -eq 0 ];then - mkdir -p ${work_dir}/build + mkdir -p ${GITHUB_WORKSPACE}/build mv ${md5_content}.tar.xz ${xz_dir} else mkdir -p ${xz_dir} echo "update_cached_package=ON" >> $GITHUB_ENV fi else - mkdir -p ${work_dir}/build + mkdir -p ${GITHUB_WORKSPACE}/build fi From fa2bbc51ccf9f190d6c4a1f8da9d2ef925329f1f Mon Sep 17 00:00:00 2001 From: gushiwei Date: Fri, 25 Apr 2025 09:42:13 +0000 Subject: [PATCH 3/4] test=document_fix rerun --- .github/workflows/_Linux-XPU.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_Linux-XPU.yml b/.github/workflows/_Linux-XPU.yml index c59e0eaf332b54..81be958517f636 100644 --- a/.github/workflows/_Linux-XPU.yml +++ b/.github/workflows/_Linux-XPU.yml @@ -131,7 +131,7 @@ jobs: ulimit -n 102400 git config --global --add safe.directory ${work_dir} git submodule foreach "git config --global --add safe.directory \$toplevel/\$sm_path" - bash -x ${ci_scripts}/run_setup.sh bdist_wheel + bash -x ${ci_scripts}/run_setup.sh bdist_wheel ' - name: Upload product to bos From a32426aac00eb17cce1120eb7f4a25a925b4f3c1 Mon Sep 17 00:00:00 2001 From: gushiwei Date: Sun, 27 Apr 2025 02:32:28 +0000 Subject: [PATCH 4/4] test=document_fix sudo rm --- .github/workflows/_Linux-XPU.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_Linux-XPU.yml b/.github/workflows/_Linux-XPU.yml index 81be958517f636..7fabaebd6841bd 100644 --- a/.github/workflows/_Linux-XPU.yml +++ b/.github/workflows/_Linux-XPU.yml @@ -151,8 +151,8 @@ jobs: mkdir ${{ env.home_path }}/bos tar xf ${{ env.home_path }}/bos_new.tar.gz -C ${{ env.home_path }}/bos fi - rm -rf `find . -name "*.a"` - rm -rf `find . -name "*.o"` + sudo rm -rf `find . -name "*.a"` + sudo rm -rf `find . -name "*.o"` cd ${{ github.workspace }}/.. tar --use-compress-program="pigz" -cpf build.tar.gz Paddle # source /home/opt/deck/1.0/etc/bashrc