Skip to content

Commit df6d78a

Browse files
risemeup1risemeup1
authored
Merge pull request #1469 from 0x3878f/pipeline
Enhance monitoring for macOS and Windows platforms using github action
2 parents df05e23 + 6bf9d26 commit df6d78a

14 files changed

+367
-277
lines changed

‎.github/workflows/build_and_test.yml

+75-51
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@ name: Build and Test
22

33
on:
44
push:
5-
branches: [ "develop" ]
5+
branches: [ "test_pir" ]
66
pull_request:
7-
branches: [ "develop" ]
7+
branches: [ "test_pir" ]
88

99
permissions:
1010
contents: read
@@ -15,20 +15,19 @@ jobs:
1515
strategy:
1616
matrix:
1717
python-version: [ '3.8']
18-
architecture: [ 'x64' ]
19-
18+
architecture: [ 'x86_64']
2019
steps:
2120
# Checkout the latest branch of Paddle2ONNX.
2221
- name: Checkout Paddle2ONNX
2322
uses: actions/checkout@v4
2423
with:
2524
submodules: true
2625

27-
- name: Build on manylinux2014_x86_64
28-
uses: docker://quay.io/pypa/manylinux2014_x86_64:latest
26+
- name: Build on manylinux_2_28_x86_64
27+
uses: docker://quay.io/pypa/manylinux_2_28_x86_64:latest
2928
with:
3029
entrypoint: bash
31-
args: .github/workflows/scripts/entrypoint.sh ${{ matrix.python-version }} manylinux2014_x86_64 CentOS
30+
args: .github/workflows/scripts/entrypoint.sh ${{ matrix.python-version }}
3231

3332
- name: Setup Python 3.8
3433
uses: actions/setup-python@v5
@@ -50,49 +49,62 @@ jobs:
5049
runs-on: windows-2019
5150
strategy:
5251
matrix:
53-
python-version: [ '3.8' ]
52+
python-version: [ '3.10' ]
5453
architecture: [ 'x64' ]
5554

5655
steps:
57-
# Checkout the latest branch of Paddle2ONNX.
58-
- name: Checkout Paddle2ONNX
59-
uses: actions/checkout@v4
60-
with:
61-
submodules: true
62-
63-
# Download and extract protobuf
64-
- name: Download and extract protobuf
65-
run: |
66-
curl -L -o protobuf.zip https://bj.bcebos.com/fastdeploy/third_libs/protobuf-win-x64-3.16.0.zip
67-
powershell -Command "Expand-Archive -Path protobuf.zip -DestinationPath $PWD/protobuf"
68-
69-
# Setup Python
70-
- name: Set up Python ${{ matrix.python-version }}
71-
uses: actions/setup-python@v5
72-
with:
73-
python-version: ${{ matrix.python-version }}
74-
# Install Python dependencies
75-
- name: Install Python dependencies
76-
run: |
77-
python -m pip install -q --upgrade pip
78-
python -m pip install setuptools wheel auditwheel auditwheel-symbols build
79-
80-
# Build package
81-
- name: Build package
82-
run: |
83-
$Env:PATH = "${{ github.workspace }}\protobuf\bin;" + $Env:PATH
84-
python -m build --wheel
85-
86-
# Install Paddle2ONNX
87-
- name: Install Paddle2ONNX
88-
run: |
89-
Get-ChildItem -Path dist/*.whl | foreach {python -m pip install --upgrade $_.fullname}
90-
91-
# Install Test
92-
- name: Run Test
93-
working-directory: ./tests
94-
run: |
95-
.\run.bat python
56+
# Checkout the latest branch of Paddle2ONNX.
57+
- name: Checkout Paddle2ONNX
58+
uses: actions/checkout@v4
59+
with:
60+
submodules: true
61+
62+
# Clone protobuf repository and checkout to v21.12
63+
- name: Clone protobuf
64+
run: |
65+
git clone https://github.com/protocolbuffers/protobuf.git
66+
cd protobuf
67+
git checkout v21.12
68+
69+
# Build and install protobuf
70+
- name: Build and install protobuf
71+
run: |
72+
cd protobuf
73+
git submodule update --init --recursive
74+
mkdir build
75+
cd build
76+
cmake -G "Visual Studio 16 2019" -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}\protobuf\protobuf_install\ -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_EXAMPLES=OFF ..
77+
cmake --build . --config Release --target install
78+
79+
# Setup Python
80+
- name: Set up Python ${{ matrix.python-version }}
81+
uses: actions/setup-python@v5
82+
with:
83+
python-version: ${{ matrix.python-version }}
84+
85+
# Install Python dependencies
86+
- name: Install Python dependencies
87+
run: |
88+
python -m pip install -q --upgrade pip
89+
python -m pip install setuptools wheel auditwheel auditwheel-symbols build
90+
91+
# Build package
92+
- name: Build package
93+
run: |
94+
$Env:PATH = "${{ github.workspace }}\protobuf\protobuf_install\bin;" + $Env:PATH
95+
$Env:PIP_EXTRA_INDEX_URL="https://www.paddlepaddle.org.cn/packages/nightly/cpu/"
96+
python -m build --wheel
97+
98+
# Install Paddle2ONNX
99+
- name: Install Paddle2ONNX
100+
run: |
101+
Get-ChildItem -Path dist/*.whl | foreach {python -m pip install --upgrade $_.fullname}
102+
103+
# Install Test
104+
- name: Run Test
105+
working-directory: ./tests
106+
run: |
107+
.\run.bat python
96108
97109
build_on_macos:
98110
# Use MACOSX_DEPLOYMENT_TARGET=12.0 to produce compatible wheel
@@ -103,7 +115,6 @@ jobs:
103115
matrix:
104116
python-version: [ '3.8']
105117
architecture: [ 'arm64' ]
106-
107118
steps:
108119
# Checkout the latest branch of Paddle2ONNX.
109120
- name: Checkout Paddle2ONNX
@@ -117,10 +128,22 @@ jobs:
117128
with:
118129
python-version: ${{ matrix.python-version }}
119130

120-
# Download protobuf
121-
- name: Download protobuf
131+
# Clone protobuf repository and checkout to v21.12
132+
- name: Clone protobuf
133+
run: |
134+
git clone https://github.com/protocolbuffers/protobuf.git
135+
cd protobuf
136+
git checkout v21.12
137+
138+
# Build and install protobuf
139+
- name: Build and install protobuf
122140
run: |
123-
source .github/workflows/scripts/download_protobuf.sh
141+
cd protobuf
142+
git submodule update --init --recursive
143+
mkdir build
144+
cd build
145+
cmake ../cmake -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed_protobuf -Dprotobuf_BUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=14
146+
cmake --build . --target install
124147
125148
# Install Python dependencies
126149
- name: Install Python dependencies
@@ -132,6 +155,7 @@ jobs:
132155
- name: Build package
133156
run: |
134157
export PATH="${{ github.workspace }}/installed_protobuf/bin:$PATH"
158+
export PIP_EXTRA_INDEX_URL="https://www.paddlepaddle.org.cn/packages/nightly/cpu/"
135159
python -m build --wheel
136160
137161
# Install Paddle2ONNX

‎.github/workflows/release_linux_aarch64.yml

+5-5
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ jobs:
1414
matrix:
1515
python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12']
1616
architecture: [ 'x64' ]
17-
17+
1818
steps:
1919
# Checkout the latest branch of Paddle2ONNX.
2020
- name: Checkout Paddle2ONNX
@@ -25,11 +25,11 @@ jobs:
2525
# setting up qemu for enabling aarch64 binary execution on x86 machine
2626
- uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
2727

28-
- name: Build on manylinux2014_aarch64
29-
uses: docker://quay.io/pypa/manylinux2014_aarch64:latest
28+
- name: Build on manylinux_2_28_aarch64
29+
uses: docker://quay.io/pypa/manylinux_2_28_aarch64
3030
with:
3131
entrypoint: bash
32-
args: .github/workflows/scripts/entrypoint.sh ${{ matrix.python-version }} manylinux2014_aarch64 CentOS
32+
args: .github/workflows/scripts/entrypoint.sh ${{ matrix.python-version }}
3333

3434
- uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
3535
with:
@@ -40,4 +40,4 @@ jobs:
4040
uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
4141
with:
4242
user: __token__
43-
password: ${{ secrets.PADDLE2ONNX_API_TOKEN }}
43+
password: ${{ secrets.PADDLE2ONNX_API_TOKEN }}

‎.github/workflows/release_linux_x86_64.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,11 @@ jobs:
2222
with:
2323
submodules: true
2424

25-
- name: Build on manylinux2014_x86_64
26-
uses: docker://quay.io/pypa/manylinux2014_x86_64:latest
25+
- name: Build on manylinux_2_28_x86_64
26+
uses: docker://quay.io/pypa/manylinux_2_28_x86_64:latest
2727
with:
2828
entrypoint: bash
29-
args: .github/workflows/scripts/entrypoint.sh ${{ matrix.python-version }} manylinux2014_x86_64 CentOS
29+
args: .github/workflows/scripts/entrypoint.sh ${{ matrix.python-version }}
3030

3131
- uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
3232
with:
@@ -37,4 +37,4 @@ jobs:
3737
uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
3838
with:
3939
user: __token__
40-
password: ${{ secrets.PADDLE2ONNX_API_TOKEN }}
40+
password: ${{ secrets.PADDLE2ONNX_API_TOKEN }}

‎.github/workflows/release_mac.yml

+16-9
Original file line numberDiff line numberDiff line change
@@ -26,22 +26,28 @@ jobs:
2626
with:
2727
submodules: true
2828

29-
# Checkout the latest branch of Paddle2ONNX.
30-
- name: Checkout Paddle2ONNX
31-
uses: actions/checkout@v4
32-
with:
33-
submodules: true
34-
3529
# Setup Python
3630
- name: Set up Python ${{ matrix.python-version }}
3731
uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
3832
with:
3933
python-version: ${{ matrix.python-version }}
4034

41-
# Download protobuf
42-
- name: Download protobuf
35+
# Clone protobuf repository and checkout to v21.12
36+
- name: Clone protobuf
37+
run: |
38+
git clone https://github.com/protocolbuffers/protobuf.git
39+
cd protobuf
40+
git checkout v21.12
41+
42+
# Build and install protobuf
43+
- name: Build and install protobuf
4344
run: |
44-
source .github/workflows/scripts/download_protobuf.sh
45+
cd protobuf
46+
git submodule update --init --recursive
47+
mkdir build
48+
cd build
49+
cmake ../cmake -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed_protobuf -Dprotobuf_BUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=14
50+
cmake --build . --target install
4551
4652
# Install Python dependencies
4753
- name: Install Python dependencies
@@ -53,6 +59,7 @@ jobs:
5359
- name: Build package
5460
run: |
5561
export PATH="${{ github.workspace }}/installed_protobuf/bin:$PATH"
62+
export PIP_EXTRA_INDEX_URL="https://www.paddlepaddle.org.cn/packages/nightly/cpu/"
5663
python -m build --wheel
5764
5865
- uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32

‎.github/workflows/release_win_amd64.yml

+20-6
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ jobs:
1212
runs-on: windows-2019
1313
strategy:
1414
matrix:
15-
python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12']
15+
# python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12']
16+
python-version: ['3.10']
1617
architecture: [ 'x64' ]
1718

1819
steps:
@@ -22,17 +23,29 @@ jobs:
2223
with:
2324
submodules: true
2425

25-
# Download and extract protobuf
26-
- name: Download and extract protobuf
26+
# Clone protobuf repository and checkout to v21.12
27+
- name: Clone protobuf
2728
run: |
28-
curl -L -o protobuf.zip https://bj.bcebos.com/fastdeploy/third_libs/protobuf-win-x64-3.16.0.zip
29-
powershell -Command "Expand-Archive -Path protobuf.zip -DestinationPath $PWD/protobuf"
29+
git clone https://github.com/protocolbuffers/protobuf.git
30+
cd protobuf
31+
git checkout v21.12
32+
33+
# Build and install protobuf
34+
- name: Build and install protobuf
35+
run: |
36+
cd protobuf
37+
git submodule update --init --recursive
38+
mkdir build
39+
cd build
40+
cmake -G "Visual Studio 16 2019" -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}\protobuf\protobuf_install\ -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_EXAMPLES=OFF ..
41+
cmake --build . --config Release --target install
3042
3143
# Setup Python
3244
- name: Set up Python ${{ matrix.python-version }}
3345
uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
3446
with:
3547
python-version: ${{ matrix.python-version }}
48+
3649
# Install Python dependencies
3750
- name: Install Python dependencies
3851
run: |
@@ -41,7 +54,8 @@ jobs:
4154
4255
- name: Build package
4356
run: |
44-
$Env:PATH = "${{ github.workspace }}\protobuf\bin;" + $Env:PATH
57+
$Env:PATH = "${{ github.workspace }}\protobuf\protobuf_install\bin;" + $Env:PATH
58+
$Env:PIP_EXTRA_INDEX_URL="https://www.paddlepaddle.org.cn/packages/nightly/cpu/"
4559
python -m build --wheel
4660
4761
- uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32

‎.github/workflows/scripts/entrypoint.sh

+31-6
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,19 @@
11
#!/bin/bash
22

3+
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
317
# Copyright (c) ONNX Project Contributors
418
#
519
# SPDX-License-Identifier: Apache-2.0
@@ -24,20 +38,31 @@ PYTHON_COMMAND="/opt/python/${PY_VER}/bin/python"
2438
$PIP_INSTALL_COMMAND --upgrade pip
2539
$PIP_INSTALL_COMMAND cmake
2640

27-
# Build protobuf from source
28-
if [[ "$SYSTEM_NAME" == "CentOS" ]]; then
29-
yum install -y wget
30-
fi
31-
source .github/workflows/scripts/download_protobuf.sh
41+
# Build and install protobuf
42+
original_dir=$(pwd)
43+
git clone https://github.com/protocolbuffers/protobuf.git
44+
cd protobuf
45+
git checkout v21.12
46+
git submodule update --init
47+
mkdir build_source && cd build_source
48+
cmake ../cmake -DCMAKE_INSTALL_PREFIX=`pwd`/installed_protobuf_lib -Dprotobuf_BUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release
49+
cmake --build . --target install
50+
export PATH=`pwd`/installed_protobuf_lib/bin:${PATH}
51+
cd $original_dir
52+
53+
export PIP_EXTRA_INDEX_URL="https://www.paddlepaddle.org.cn/packages/nightly/cpu/"
3254

3355
# Build Paddle2ONNX wheels
3456
$PYTHON_COMMAND -m build --wheel || { echo "Building wheels failed."; exit 1; }
3557

58+
# ============================================================================
59+
# Due to libpaddle’s limitations, it can only maintain a restricted platform tag of ‘linux_x86_64’.
60+
# ============================================================================
3661
# Bundle external shared libraries into the wheels
3762
# find -exec does not preserve failed exit codes, so use an output file for failures
3863
failed_wheels=$PWD/failed-wheels
3964
rm -f "$failed_wheels"
40-
find . -type f -iname "*-linux*.whl" -exec sh -c "auditwheel repair '{}' -w \$(dirname '{}') --plat '${PLAT}' || { echo 'Repairing wheels failed.'; auditwheel show '{}' >> '$failed_wheels'; }" \;
65+
find . -type f -iname "*-linux*.whl" -exec sh -c "auditwheel repair '{}' -w \$(dirname '{}') --exclude libpaddle.so || { echo 'Repairing wheels failed.'; auditwheel show '{}' >> '$failed_wheels'; }" \;
4166

4267
if [[ -f "$failed_wheels" ]]; then
4368
echo "Repairing wheels failed:"

‎VERSION_NUMBER

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
1.3.1
1+
2.0.0

0 commit comments

Comments
 (0)