diff --git a/.github/workflows/_build_linux.yml b/.github/workflows/_build_linux.yml index cb02c64ecc..7c8fb23f4e 100644 --- a/.github/workflows/_build_linux.yml +++ b/.github/workflows/_build_linux.yml @@ -124,6 +124,7 @@ jobs: echo "Date Only: $DATE_ONLY" export FASTDEPLOY_VERSION="${FASTDEPLOY_VERSION}.dev${DATE_ONLY}" fi + python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/ pip config set global.index-url http://pip.baidu.com/root/baidu/+simple/ pip config set install.trusted-host pip.baidu.com pip config set global.extra-index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple @@ -131,7 +132,6 @@ jobs: python -m pip install --upgrade pip python -m pip install -r requirements.txt python -m pip install wheel - python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/ # 编译RDMA export ENABLE_FD_RDMA=1 bash build.sh 1 python false [${COMPILE_ARCH}] diff --git a/.github/workflows/_pre_ce_test.yml b/.github/workflows/_pre_ce_test.yml new file mode 100644 index 0000000000..629075bf13 --- /dev/null +++ b/.github/workflows/_pre_ce_test.yml @@ -0,0 +1,104 @@ +name: Pre-CE-Test + +on: + workflow_call: + inputs: + DOCKER_IMAGE: + description: "Build Images" + required: true + type: string + default: "ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:fastdeploy-ciuse-cuda126" + FASTDEPLOY_ARCHIVE_URL: + description: "URL of the compressed FastDeploy code archive." + required: true + type: string + FASTDEPLOY_WHEEL_URL: + description: "URL of the FastDeploy Wheel." + required: true + type: string + CACHE_DIR: + description: "Cache Dir Use" + required: false + type: string + default: "" + +concurrency: + group: ${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + run_ce_cases: + runs-on: [self-hosted, GPU-L20-4Card] + steps: + - name: Print current runner name + run: | + echo "Current runner name: ${{ runner.name }}" + - name: Code Prepare + shell: bash + env: + docker_image: ${{ inputs.DOCKER_IMAGE }} + fd_archive_url: ${{ inputs.FASTDEPLOY_ARCHIVE_URL }} + run: | + set -x + REPO="https://github.com/${{ github.repository }}.git" + FULL_REPO="${{ github.repository }}" + REPO_NAME="${FULL_REPO##*/}" + BASE_BRANCH="${{ github.base_ref }}" + + # Clean the repository directory before starting + docker run --rm --net=host -v $(pwd):/workspace -w /workspace \ + -e "REPO_NAME=${REPO_NAME}" \ + ${docker_image} /bin/bash -c ' + if [ -d ${REPO_NAME} ]; then + echo "Directory ${REPO_NAME} exists, removing it..." + rm -rf ${REPO_NAME}* + fi + ' + + wget -q ${fd_archive_url} + tar -xf FastDeploy.tar.gz + rm -rf FastDeploy.tar.gz + cd FastDeploy + git config --global user.name "FastDeployCI" + git config --global user.email "fastdeploy_ci@example.com" + git log -n 3 --oneline + + - name: Run CI unittest + env: + docker_image: ${{ inputs.DOCKER_IMAGE }} + fd_wheel_url: ${{ inputs.FASTDEPLOY_WHEEL_URL }} + run: | + runner_name="${{ runner.name }}" + last_char="${runner_name: -1}" + + if [ "${last_char}" = "1" ]; then + gpu_id=2 + DEVICES="2,3" + else + gpu_id=0 + DEVICES="0,1" + fi + FD_API_PORT=$((9180 + gpu_id * 100)) + FD_ENGINE_QUEUE_PORT=$((9150 + gpu_id * 100)) + FD_METRICS_PORT=$((9170 + gpu_id * 100)) + + PARENT_DIR=$(dirname "$WORKSPACE") + echo "PARENT_DIR:$PARENT_DIR" + docker run --rm --net=host -v $(pwd):/workspace -w /workspace \ + -v "/ssd4/GithubActions/gitconfig:/etc/gitconfig:ro" \ + -v "/ssd4/GithubActions/ModelData:/ModelData:ro" \ + -v "/ssd4/GithubActions/CacheDir:/root/.cache" \ + -v "/ssd4/GithubActions/ConfigDir:/root/.config" \ + -e "MODEL_PATH=/ModelData" \ + -e "FD_API_PORT=${FD_API_PORT}" \ + -e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \ + -e "FD_METRICS_PORT=${FD_METRICS_PORT}" \ + -e "fd_wheel_url=${fd_wheel_url}" \ + --gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -c ' + git config --global --add safe.directory /workspace/FastDeploy + cd FastDeploy + # python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/ + python -m pip install paddlepaddle-gpu==3.0.0.dev20250729 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/ + python -m pip install ${fd_wheel_url} + bash scripts/run_pre_ce.sh + ' diff --git a/.github/workflows/_unit_test_coverage.yml b/.github/workflows/_unit_test_coverage.yml index 17b742cfe0..17243768ab 100644 --- a/.github/workflows/_unit_test_coverage.yml +++ b/.github/workflows/_unit_test_coverage.yml @@ -25,10 +25,11 @@ on: jobs: run_tests_with_coverage: - runs-on: [self-hosted, GPU-h1z1-4Cards] + runs-on: [self-hosted, GPU-h1z1-2Cards] outputs: diff_cov_file_url: ${{ steps.cov_upload.outputs.diff_cov_file_url }} - unittest_failed_url: ${{ steps.unittest_failed.outputs.unittest_failed_url }} + unittest_failed_url: ${{ steps.cov_upload.outputs.unittest_failed_url }} + diff_cov_result_json_url: ${{ steps.cov_upload.outputs.diff_cov_result_json_url }} steps: - name: Code Prepare shell: bash @@ -111,7 +112,7 @@ jobs: coverage combine coveragedata/ coverage xml -o python_coverage_all.xml COVERAGE_EXIT_CODE=0 - diff-cover python_coverage_all.xml --diff-file=diff.txt --fail-under=90 || COVERAGE_EXIT_CODE=9 + diff-cover python_coverage_all.xml --diff-file=diff.txt --fail-under=80 --json-report diff_coverage.json || COVERAGE_EXIT_CODE=9 echo "COVERAGE_EXIT_CODE=${COVERAGE_EXIT_CODE}" >> exit_code.env python scripts/generate_diff_coverage_xml.py diff.txt python_coverage_all.xml ' @@ -125,27 +126,68 @@ jobs: cd FastDeploy commit_id=${{ github.event.pull_request.head.sha }} pr_num=${{ github.event.pull_request.number }} - target_path=paddle-github-action/PR/FastDeploy/${pr_num}/${commit_id}/SM${compile_arch//,/_}/CoverageData + target_path=paddle-github-action/PR/FastDeploy/${pr_num}/${commit_id}/SM${compile_arch//,/_} wget -q --no-proxy --no-check-certificate https://paddle-qa.bj.bcebos.com/CodeSync/develop/PaddlePaddle/PaddleTest/tools/bos_tools.py push_file=$(realpath bos_tools.py) python -m pip install bce-python-sdk==0.9.29 diff_cov_file="diff_coverage.xml" if [ -f ${diff_cov_file} ];then - python ${push_file} ${diff_cov_file} ${target_path} + python ${push_file} ${diff_cov_file} ${target_path}/CoverageData target_path_stripped="${target_path#paddle-github-action/}" - DIFF_COV_FILE_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/${diff_cov_file} + DIFF_COV_FILE_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/CoverageData/${diff_cov_file} echo "diff_cov_file_url=${DIFF_COV_FILE_URL}" >> $GITHUB_OUTPUT + echo "diff_cov_file_url=${DIFF_COV_FILE_URL}" >> $GITHUB_ENV fi - - name: Determine Unit Succ and whether the coverage rate reaches 90% + diff_cov_result_json="diff_coverage.json" + if [ -f ${diff_cov_result_json} ];then + python ${push_file} ${diff_cov_result_json} ${target_path}/CoverageData + target_path_stripped="${target_path#paddle-github-action/}" + DIFF_COV_JSON_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/CoverageData/${diff_cov_result_json} + echo "diff_cov_result_json_url=${DIFF_COV_JSON_URL}" >> $GITHUB_OUTPUT + echo "diff_cov_result_json_url=${DIFF_COV_JSON_URL}" >> $GITHUB_ENV + fi + unittest_result="test/failed_tests.log" + if [ -s ${unittest_result} ];then + python ${push_file} ${unittest_result} ${target_path}/UnitTestResult + target_path_stripped="${target_path#paddle-github-action/}" + UNIT_TEST_RESULT_URL=https://paddle-github-action.bj.bcebos.com/${target_path_stripped}/UnitTestResult/${unittest_result} + echo "unittest_failed_url=${UNIT_TEST_RESULT_URL}" >> $GITHUB_OUTPUT + echo "unittest_failed_url=${UNIT_TEST_RESULT_URL}" >> $GITHUB_ENV + fi + - name: Determine Unit Succ and whether the coverage rate reaches 80% shell: bash run: | if [ "$TEST_EXIT_CODE" -eq 8 ]; then + if [ -z "${unittest_failed_url}" ]; then + echo "No diff unit failed file URL provided." + else + wget ${unittest_failed_url} || echo "Download unittest file failed, but continuing..." + fi echo "Unit tests failed (exit code 8)" + filename=$(basename "$unittest_failed_url") + if [ -f "${filename}" ];then + echo "Failed test cases:" + cat "${filename}" + fi exit "$TEST_EXIT_CODE" fi if [ "$COVERAGE_EXIT_CODE" -eq 9 ]; then echo "Coverage generation failed (exit code 9)" + if [ -z "${diff_cov_result_json_url}" ]; then + echo "No diff cov result file URL provided." + else + wget ${diff_cov_result_json_url} || echo "Download cov json file failed, but continuing..." + fi + filename=$(basename "$diff_cov_result_json_url") + if [ -f "${filename}" ];then + echo "Failed test cases:" + if command -v jq >/dev/null 2>&1; then + jq . "${filename}" + else + cat "${filename}" + fi + fi exit "$COVERAGE_EXIT_CODE" fi echo "All tests and coverage passed" diff --git a/.github/workflows/approve.yml b/.github/workflows/approve.yml index bf82f82000..baa953ab5a 100644 --- a/.github/workflows/approve.yml +++ b/.github/workflows/approve.yml @@ -33,7 +33,6 @@ jobs: uses: actions/setup-python@v5 with: python-version: '3.10' - cache: 'pip' - name: Run approval check script run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 518b15eb99..b07d099de1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,77 +13,8 @@ concurrency: jobs: build: - runs-on: [self-hosted, GPU-L20-4Card] + runs-on: ubuntu-latest steps: - name: Print current runner name run: | - echo "Current runner name: ${{ runner.name }}" - # Because the system version is lower than 2.23, the checkout cannot be used. - # - name: Checkout code - # uses: actions/checkout@v4 - - - name: Code Checkout - env: - docker_image: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:fastdeploy-ciuse-cuda126 - run: | - REPO="https://github.com/${{ github.repository }}.git" - FULL_REPO="${{ github.repository }}" - REPO_NAME="${FULL_REPO##*/}" - BASE_BRANCH="${{ github.base_ref }}" - # Clean the repository directory before starting - docker run --rm --net=host -v $(pwd):/workspace -w /workspace \ - -e "REPO_NAME=${REPO_NAME}" \ - -e "BASE_BRANCH=${BASE_BRANCH}" \ - ${docker_image} /bin/bash -c ' - if [ -d ${REPO_NAME} ]; then - echo "Directory ${REPO_NAME} exists, removing it..." - rm -rf ${REPO_NAME} - fi - ' - git config --global user.name "FastDeployCI" - git config --global user.email "fastdeploy_ci@example.com" - git clone ${REPO} ${REPO_NAME} -b ${BASE_BRANCH} - cd FastDeploy - if [ "${{ github.event_name }}" = "pull_request" ]; then - git fetch origin pull/${{ github.event.pull_request.number }}/head:pr/${{ github.event.pull_request.number }} - git merge pr/${{ github.event.pull_request.number }} - git log -n 3 --oneline - else - git checkout ${{ github.sha }} - git log -n 3 --oneline - fi - - - name: Run CI unittest - env: - docker_image: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:fastdeploy-ciuse-cuda126 - run: | - runner_name="${{ runner.name }}" - last_char="${runner_name: -1}" - - if [ "${last_char}" = "1" ]; then - gpu_id=2 - DEVICES="2,3" - else - gpu_id=0 - DEVICES="0,1" - fi - FD_API_PORT=$((9180 + gpu_id * 100)) - FD_ENGINE_QUEUE_PORT=$((9150 + gpu_id * 100)) - FD_METRICS_PORT=$((9170 + gpu_id * 100)) - - PARENT_DIR=$(dirname "$WORKSPACE") - echo "PARENT_DIR:$PARENT_DIR" - docker run --rm --net=host -v $(pwd):/workspace -w /workspace \ - -v "/ssd4/GithubActions/gitconfig:/etc/gitconfig:ro" \ - -v "/ssd4/GithubActions/ModelData:/ModelData:ro" \ - -v "/ssd4/GithubActions/CacheDir:/root/.cache" \ - -v "/ssd4/GithubActions/ConfigDir:/root/.config" \ - -e "MODEL_PATH=/ModelData" \ - -e "FD_API_PORT=${FD_API_PORT}" \ - -e "FD_ENGINE_QUEUE_PORT=${FD_ENGINE_QUEUE_PORT}" \ - -e "FD_METRICS_PORT=${FD_METRICS_PORT}" \ - --gpus '"device='"${DEVICES}"'"' ${docker_image} /bin/bash -c " - git config --global --add safe.directory /workspace/FastDeploy - cd FastDeploy - bash scripts/run_ci.sh - " + echo "The current CI tasks have been migrated to PreCe and will be deprecated soon." diff --git a/.github/workflows/pr_build_and_test.yml b/.github/workflows/pr_build_and_test.yml index d6557fc625..0123e5a554 100644 --- a/.github/workflows/pr_build_and_test.yml +++ b/.github/workflows/pr_build_and_test.yml @@ -21,7 +21,7 @@ jobs: with: DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddleqa:cuda126-py310 FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} - COMPILE_ARCH: "90" + COMPILE_ARCH: "89,90" WITH_NIGHTLY_BUILD: "OFF" FD_VERSION: "0.0.0" @@ -52,3 +52,12 @@ jobs: PADDLETEST_ARCHIVE_URL: "https://xly-devops.bj.bcebos.com/PaddleTest/PaddleTest.tar.gz" FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }} MODEL_CACHE_DIR: "/ssd2/actions-runner/ModelCache" + + pre_ce_test: + name: Extracted partial CE model tasks to run in CI. + needs: [clone,build] + uses: ./.github/workflows/_pre_ce_test.yml + with: + DOCKER_IMAGE: ccr-2vdh3abv-pub.cnc.bj.baidubce.com/paddlepaddle/paddle:fastdeploy-ciuse-cuda126 + FASTDEPLOY_ARCHIVE_URL: ${{ needs.clone.outputs.repo_archive_url }} + FASTDEPLOY_WHEEL_URL: ${{ needs.build.outputs.wheel_path }} diff --git a/scripts/coverage_run.sh b/scripts/coverage_run.sh index 98ed025bdc..6b6cbbf850 100644 --- a/scripts/coverage_run.sh +++ b/scripts/coverage_run.sh @@ -6,7 +6,23 @@ run_path="$DIR/../test/" cd ${run_path} ls -dirs=("layers" "operators" "worker" "utils") +exclude=("ci_use" "ce") +for d in */ ; do + dir_name="${d%/}" + if [[ -d "$dir_name" ]]; then + skip=false + for ex in "${exclude[@]}"; do + if [[ "$dir_name" == "$ex" ]]; then + skip=true + break + fi + done + if ! $skip; then + dirs+=("$dir_name") + fi + fi +done + failed_tests_file="failed_tests.log" > "$failed_tests_file" disabled_tests=( @@ -20,6 +36,10 @@ disabled_tests=( operators/test_stop_generation.py operators/test_air_topp_sampling.py operators/test_fused_moe.py + layers/test_repetition_early_stopper.py + operators/test_stop_generation_multi_ends.py + utils/test_download.py + graph_optimization/test_cuda_graph.py ) is_disabled() { local test_file_rel="$1" diff --git a/scripts/run_ci.sh b/scripts/run_pre_ce.sh similarity index 94% rename from scripts/run_ci.sh rename to scripts/run_pre_ce.sh index 91ef179b75..726b91e857 100644 --- a/scripts/run_ci.sh +++ b/scripts/run_pre_ce.sh @@ -3,13 +3,10 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" echo "$DIR" # python -m pip install --pre paddlepaddle-gpu -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/ -python -m pip install paddlepaddle-gpu==3.0.0.dev20250729 -i https://www.paddlepaddle.org.cn/packages/nightly/cu126/ python -m pip config set global.index-url https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple - python -m pip install -r requirements.txt python -m pip install jsonschema aistudio_sdk==0.3.5 -bash build.sh || exit 1 failed_files=() run_path="$DIR/../test/ci_use/"