CI Support LinuxRelease_aarch64 (#1283)

Author: Zheng-Bicheng

.github/workflows/build_and_test.yml

@@ -53,7 +53,6 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install setuptools wheel auditwheel auditwheel-symbols build
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
 
       # Build Paddle2ONNX
       - name: Build Paddle2ONNX

.github/workflows/release_linux_aarch64.yml

@@ -0,0 +1,44 @@
+name: LinuxRelease_aarch64
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: [ '3.8']
+        architecture: [ 'x64' ]
+    
+    steps:
+      # Checkout the latest branch of Paddle2ONNX.
+      - name: Checkout Paddle2ONNX
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      # setting up qemu for enabling aarch64 binary execution on x86 machine
+      - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
+
+      - name: Build on manylinux2014_aarch64
+        uses: docker://quay.io/pypa/manylinux2014_aarch64:latest
+        with:
+          entrypoint: bash
+          args: .github/workflows/scripts/entrypoint.sh ${{ matrix.python-version }} manylinux2014_aarch64 CentOS
+
+      - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
+        with:
+          name: wheels
+          path: dist
+
+      - name: Publish package
+        uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+        with:
+          user: __token__
+          password: ${{ secrets.ZHENG_BICHENG_PYPI_TOKEN }}

.github/workflows/release_linux_x86_64.yml

@@ -1,4 +1,4 @@
-name: Build Package
+name: LinuxRelease_x86_64
 
 on:
   release:
@@ -12,11 +12,10 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12']
+        # python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: [ '3.8']
         architecture: [ 'x64' ]
 
-    # For the sake of simplicity in testing, the Paddle2ONNX packaging program will temporarily only package executable files for Python 3.8.
-    # In the future, we will need to extend support to cover Python 3.8 through Python 3.10.
     steps:
       # Checkout the latest branch of Paddle2ONNX.
       - name: Checkout Paddle2ONNX
@@ -28,7 +27,7 @@ jobs:
         uses: docker://quay.io/pypa/manylinux2014_x86_64:latest
         with:
           entrypoint: bash
-          args: .github/workflows/scripts/entrypoint.sh ${{ matrix.python-version }} manylinux2014_x86_64
+          args: .github/workflows/scripts/entrypoint.sh ${{ matrix.python-version }} manylinux2014_x86_64 CentOS
 
       - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
         with:
@@ -39,4 +38,4 @@ jobs:
         uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
         with:
           user: __token__
-          password: ${{ secrets.ZHENG_BICHENG_PYPI_TOKEN }}
+          password: ${{ secrets.PADDLE2ONNX_API_TOKEN }}

.github/workflows/scripts/build_protobuf_unix.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Detect the operating system
+OS=$(uname -s)
+ARCH=$(uname -m)
+
+# Check if the operating system is Linux
+if [ "$OS" = "Linux" ]; then
+    if [[ "$ARCH" == "x86_64" ]]; then
+      protobuf_tgz_name="protobuf-linux-x64-3.16.0.tgz"
+    elif [[ "$ARCH" == "arm"* || "$ARCH" == "aarch64" ]]; then
+      protobuf_tgz_name="protobuf-linux-aarch64-3.16.0.tgz"
+    else
+        echo "When the operating system is Linux, the system architecture only supports (x86_64 and aarch64), but the current architecture is $ARCH."
+        exit 1
+    fi
+# Check if the operating system is Darwin (macOS)
+elif [ "$OS" = "Darwin" ]; then
+    if [[ "$ARCH" == "x86_64" ]]; then
+      protobuf_tgz_name="protobuf-osx-x86_64-3.16.0.tgz"
+    elif [[ "$ARCH" == "arm64" ]]; then
+      protobuf_tgz_name="protobuf-osx-arm64-3.16.0.tgz"
+    else
+      echo "When the operating system is Darwin, the system architecture only supports (x86_64 and arm64), but the current architecture is $ARCH."
+      exit 1
+    fi
+else
+   echo "The system only supports (Linux and Darwin), but the current system is $OS."
+fi
+
+protobuf_url="https://bj.bcebos.com/paddle2onnx/third_party/$protobuf_tgz_name"
+wget $protobuf_url
+protobuf_svae_dir="$PWD/installed_protobuf"
+mkdir -p $protobuf_svae_dir
+tar -zxf $protobuf_tgz_name -C $protobuf_svae_dir
+export PATH=$protobuf_svae_dir/bin:${PATH}

.github/workflows/scripts/download_protobuf.sh

@@ -9,28 +9,26 @@ set -e -x
 # CLI arguments
 PY_VERSION=$1
 PLAT=$2
-GITHUB_EVENT_NAME=$3
+SYSTEM_NAME=$3
 
 export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
 
 # Compile wheels
 # Need to be updated if there is a new Python Version
-if [ "$(uname -m)" == "aarch64" ]; then
- PIP_INSTALL_COMMAND="$PY_VERSION -m pip install --no-cache-dir -q"
- PYTHON_COMMAND="$PY_VERSION"
-else
- declare -A python_map=( ["3.8"]="cp38-cp38" ["3.9"]="cp39-cp39" ["3.10"]="cp310-cp310" ["3.11"]="cp311-cp311" ["3.12"]="cp312-cp312")
- PY_VER=${python_map[$PY_VERSION]}
- PIP_INSTALL_COMMAND="/opt/python/${PY_VER}/bin/pip install --no-cache-dir -q"
- PYTHON_COMMAND="/opt/python/${PY_VER}/bin/python"
-fi
+declare -A python_map=( ["3.8"]="cp38-cp38" ["3.9"]="cp39-cp39" ["3.10"]="cp310-cp310" ["3.11"]="cp311-cp311" ["3.12"]="cp312-cp312")
+PY_VER=${python_map[$PY_VERSION]}
+PIP_INSTALL_COMMAND="/opt/python/${PY_VER}/bin/pip install --no-cache-dir -q"
+PYTHON_COMMAND="/opt/python/${PY_VER}/bin/python"
 
 # Update pip and install cmake
 $PIP_INSTALL_COMMAND --upgrade pip
 $PIP_INSTALL_COMMAND cmake
 
 # Build protobuf from source
-source .github/workflows/scripts/build_protobuf_unix.sh "$(nproc)"
+if [[ "$SYSTEM_NAME" == "CentOS" ]]; then
+    yum install -y wget
+fi
+source .github/workflows/scripts/download_protobuf.sh
 
 # Build Paddle2ONNX wheels
 $PYTHON_COMMAND -m build --wheel || { echo "Building wheels failed."; exit 1; }

.github/workflows/scripts/entrypoint.sh

@@ -1,26 +1,22 @@
+# System-generated temporary files
 .DS_Store
 
-*~
+# Temporary files that may be generated when loading Paddle2ONNX using an IDE
+.idea
+.vscode
 
-*.pyc
-.pydevproject
-.eggs/*
-dist/*
-.setuptools*
-paddle2onnx.egg-info/*
-*.pdmodel
-*_*.onnx
-*.log
+# Files that change dynamically when compiling Paddle2ONNX
 version.py
 paddle2onnx/mappers_registry.h
 
-# CMD
-build/*
-paddle2onnx-*
-
-# Clion
+# Temporary files that may be generated when building Paddle2ONNX
+protobuf-*
+installed_protobuf
+build
 cmake-build-*
-.idea
+paddle2onnx.egg-info/*
+dist/*
 
-# VSCode
-.vscode
+# Temporary files automatically generated when executing Paddle2ONNX unit tests
+*.pdmodel
+*.onnx

.gitignore

@@ -1 +1 @@
-1.2.3
+1.2.4

VERSION_NUMBER

@@ -1,26 +1,70 @@
 # Docker编译安装Paddle2ONNX
 
-Paddle2ONNX编译安装需要确保环境满足以下需求
-- cmake >= 3.18.0
-- protobuf == 3.16.0
+## 1 拉取manylinux镜像
 
-注意：Paddle2ONNX产出的模型，在使用ONNX Runtime推理时，要求使用最新版本(1.10.0版本以及上），如若需要使用低版本(1.6~1.10之间），则需要将ONNX版本降至1.8.2，在执行完`git submodule update`后，执行如下命令，然后再进行编译
+根据系统架构拉取不同的manylinux镜像
+
+```bash
+# Pull manylinux2014_x86_64
+docker pull quay.io/pypa/manylinux2014_x86_64
+docker create --name p2o_build -it quay.io/pypa/manylinux2014_x86_64 /bin/bash
+# Pull manylinux2014_x86_64
+docker pull quay.io/pypa/manylinux2014_aarch64
+docker create --name p2o_build -it quay.io/pypa/manylinux2014_aarch64 /bin/bash
 ```
-cd Paddle2ONNX/third/onnx
-git checkout v1.8.1
+
+## 2 创建并进入容器
+
+创建并进入 Docker 容器
+
+```bash
+docker start p2o_build
+docker exec -it p2o_build /bin/bash
 ```
 
-拉取manylinux镜像并创建容器
+## 3 拉取 Paddle2ONNX 仓库
+
+执行以下命令来拉取并初始化 Paddle2ONNX 仓库
 
 ```bash
-docker pull quay.io/pypa/manylinux2014_x86_64
-docker run --name p2o_build  -d quay.io/pypa/manylinux2014_x86_64 
+git clone https://github.com/PaddlePaddle/Paddle2ONNX.git
+cd Paddle2ONNX
+git submodule init
+git submodule update
+```
+
+## 4 获取 protobuf 依赖库
+
+### 4.1 使用 protobuf 预编译库
+
+执行以下命令来下载 protobuf 依赖库
+
+```bash
+source .github/workflows/scripts/download_protobuf.sh
 ```
 
-创建容器并运行
+### 4.2 下载并编译 protobuf 预编译库
+
+执行以下命令来下载并编译 protobuf 预编译库
 
 ```bash
-docker start p2o_build
-docker exec -it p2o_build bash
+git clone https://github.com/protocolbuffers/protobuf.git
+cd protobuf
+git checkout v3.16.0
+mkdir build_source && cd build_source
+cmake ../cmake -DCMAKE_INSTALL_PREFIX=${PWD}/installed_protobuf_lib -Dprotobuf_BUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release
+make -j8
+make install
+
+# 将编译目录加入环境变量
+export PATH=${PWD}/installed_protobuf_lib/bin:${PATH}
 ```
 
+## 5 执行编译和安装
+
+```bash
+/opt/python/cp38-cp38/bin/pip install setuptools wheel auditwheel auditwheel-symbols build
+cd /path/to/Paddle2ONNX
+/opt/python/cp38-cp38/bin/python -m build
+/opt/python/cp38-cp38/bin/pip install dist/*.whl
+```