Skip to content

Commit 6cc3784

Browse files
tianshuo78520atianshuo78520a
authored
Update ubuntu dockerfile (#72960)
* Update ubuntu dockerfile * update dockerfile
1 parent 451cd22 commit 6cc3784

File tree

5 files changed

+338
-100
lines changed

5 files changed

+338
-100
lines changed

‎tools/dockerfile/Dockerfile.develop.npu

Lines changed: 84 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -1,106 +1,90 @@
1-
# Docker Image for PaddlePaddle NPU
1+
# Docker Image for PaddlePaddle Ascend NPU
22

3-
FROM ubuntu:20.04
3+
FROM <baseimg>
44
LABEL maintainer="PaddlePaddle Authors <paddle-dev@baidu.com>"
55

6-
ENV DEBIAN_FRONTEND=noninteractive
7-
8-
RUN apt-get update && apt-get install -y apt-utils
9-
RUN ln -snf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
10-
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata
11-
RUN apt-get update && apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa && add-apt-repository ppa:ubuntu-toolchain-r/test
12-
RUN apt-get update && apt-get install -y curl wget vim git unzip unrar tar ntp xz-utils libssl-dev bzip2 gzip make automake \
13-
coreutils language-pack-zh-hans libsm6 libxext6 libxrender-dev libgl1-mesa-glx libsqlite3-dev libopenblas-dev liblapack3 \
14-
bison libjpeg-dev zlib1g zlib1g-dev swig locales net-tools libtool numactl libnuma-dev liblzma-dev libbz2-dev libblas-dev \
15-
openssl openssh-server libffi-dev pciutils libblas3 liblapack-dev libzstd-dev default-jre libgcc-s1 gcc g++ gfortran gdb
16-
17-
# workdir
18-
WORKDIR /opt
19-
20-
# GCC 8.4
21-
RUN apt-get install -y gcc-8 g++-8 gfortran-8
22-
RUN update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-8 90 && \
23-
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 90 && \
24-
update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-8 90
25-
26-
# cmake 3.27.7
27-
RUN wget -q https://cmake.org/files/v3.27/cmake-3.27.7-linux-$(uname -m).sh && \
28-
chmod +x cmake-3.27.7-linux-$(uname -m).sh && mkdir -p /opt/cmake-3.27.7 && \
29-
./cmake-3.27.7-linux-$(uname -m).sh --prefix=/opt/cmake-3.27.7 --skip-license && \
30-
rm -rf cmake-3.27.7-linux-$(uname -m).sh
31-
ENV PATH=/opt/cmake-3.27.7/bin:${PATH}
32-
33-
# default python version
34-
ARG PY_VERSION=3.10
35-
RUN apt-get install -y python3-distutils python3.8 python3.8-dev python3.9 python3.9-dev python3.10 python3.10-dev
36-
37-
# install pip
38-
RUN curl -s -q https://bootstrap.pypa.io/get-pip.py | /usr/bin/python3.8 && \
39-
curl -s -q https://bootstrap.pypa.io/get-pip.py | /usr/bin/python3.9 && \
40-
curl -s -q https://bootstrap.pypa.io/get-pip.py | /usr/bin/python3.10
41-
42-
# set default python
43-
RUN rm -rf /usr/bin/python3 && ln -s /usr/bin/python${PY_VERSION} /usr/bin/python3 && \
44-
rm -rf /usr/bin/python && ln -s /usr/bin/python${PY_VERSION} /usr/bin/python
45-
46-
# install pylint and pre-commit
47-
RUN pip3.8 install pre-commit==2.17.0 pylint pytest astroid isort coverage qtconsole distro && \
48-
pip3.9 install pre-commit==2.17.0 pylint pytest astroid isort coverage qtconsole distro && \
49-
pip3.10 install pre-commit==2.17.0 pylint pytest astroid isort coverage qtconsole distro
50-
RUN pip3.8 install attrs pyyaml pathlib2 scipy requests psutil Cython clang-format==13.0.0 && \
51-
pip3.9 install attrs pyyaml pathlib2 scipy requests psutil Cython clang-format==13.0.0 && \
52-
pip3.10 install attrs pyyaml pathlib2 scipy requests psutil Cython clang-format==13.0.0
53-
54-
# add more libs
55-
RUN apt-get update && apt-get install libprotobuf-dev protobuf-compiler libprotoc-dev lsof libgeos-dev \
56-
pkg-config libhdf5-103 libhdf5-dev lrzsz libsndfile1 tree ninja-build -y
57-
58-
# install Paddle requirement
59-
RUN wget --no-check-certificate https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/requirements.txt -O requirements.txt && \
60-
pip3.8 install -r requirements.txt && \
61-
pip3.9 install -r requirements.txt && \
62-
pip3.10 install -r requirements.txt && rm -rf requirements.txt
63-
64-
RUN wget --no-check-certificate https://raw.githubusercontent.com/PaddlePaddle/Paddle/develop/python/unittest_py/requirements.txt -O requirements.txt && \
65-
pip3.8 install -r requirements.txt && \
66-
pip3.9 install -r requirements.txt && \
67-
pip3.10 install -r requirements.txt && rm -rf requirements.txt
68-
69-
# git credential to skip password typing
70-
RUN git config --global credential.helper store
71-
72-
# Fix locales to en_US.UTF-8
73-
RUN localedef -i en_US -f UTF-8 en_US.UTF-8
74-
75-
# patchelf 0.14.5 - https://github.com/NixOS/patchelf/pull/216
76-
RUN wget -q --no-check-certificate https://github.com/NixOS/patchelf/archive/refs/tags/0.14.5.tar.gz && \
77-
tar xzf 0.14.5.tar.gz && cd patchelf-0.14.5 && \
78-
./bootstrap.sh > /dev/null && ./configure > /dev/null && \
79-
make -j16 > /dev/null && make install > /dev/null && \
80-
cd .. && rm -rf patchelf-0.14.5 && rm -rf 0.14.5.tar.gz
81-
82-
# ccache 4.6.3
83-
RUN wget -q https://github.com/ccache/ccache/releases/download/v4.6.3/ccache-4.6.3.tar.gz && \
84-
tar xf ccache-4.6.3.tar.gz && mkdir /usr/local/ccache-4.6.3 && cd ccache-4.6.3 && \
85-
mkdir build && cd build && \
86-
cmake -DCMAKE_BUILD_TYPE=Release -DREDIS_STORAGE_BACKEND=OFF \
87-
-DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.6.3 .. > /dev/null && \
88-
make -j16 > /dev/null && make install > /dev/null && \
89-
cd ../../ && rm -rf ccache-4.6.3.tar.gz && rm -rf ccache-4.6.3 && \
90-
ln -s /usr/local/ccache-4.6.3/bin/ccache /usr/local/bin/ccache
91-
92-
ENV CCACHE_MAXSIZE=80G \
93-
CCACHE_LIMIT_MULTIPLE=0.8 \
94-
CCACHE_SLOPPINESS=clang_index_store,time_macros,include_file_mtime
95-
96-
# Configure OpenSSH server. c.f. https://docs.docker.com/engine/examples/running_ssh_service
97-
RUN mkdir /var/run/sshd && echo 'root:root' | chpasswd && \
98-
sed -ri 's/^PermitRootLogin\s+.*/PermitRootLogin yes/' /etc/ssh/sshd_config && \
99-
sed -ri 's/UsePAM yes/#UsePAM yes/g' /etc/ssh/sshd_config
100-
CMD source ~/.bashrc
101-
102-
# /proc/sys/kernel/core_pattern
103-
RUN mkdir -p /var/core
6+
7+
8+
# HwHiAiUser
9+
RUN groupadd -g 1000 HwHiAiUser && \
10+
useradd -u 1000 -g 1000 -m -d /home/HwHiAiUser HwHiAiUser
11+
12+
RUN mkdir -p /usr/local/Ascend/driver
13+
WORKDIR /usr/local/Ascend
14+
ARG CANN_VERSION
15+
ARG SYSTEM
16+
ARG NPU_VERSION
17+
18+
# install CANN requirement
19+
# https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/700alpha003/softwareinstall/instg/instg_0026.html
20+
RUN apt-get update -y && apt-get install -y zlib1g zlib1g-dev libsqlite3-dev openssl libssl-dev libffi-dev libbz2-dev \
21+
libxslt1-dev unzip pciutils net-tools libblas-dev gfortran libblas3 liblapack-dev liblapack3 libopenblas-dev
22+
23+
RUN pip3.8 install --upgrade pip setuptools wheel && \
24+
pip3.9 install --upgrade pip setuptools wheel && \
25+
pip3.10 install --upgrade pip setuptools wheel
26+
27+
RUN pip3.8 install 'numpy>=1.19.2' 'decorator>=4.4.0' 'sympy>=1.5.1' 'cffi>=1.12.3' 'protobuf>=3.13.0' && \
28+
pip3.9 install 'numpy>=1.19.2' 'decorator>=4.4.0' 'sympy>=1.5.1' 'cffi>=1.12.3' 'protobuf>=3.13.0' && \
29+
pip3.10 install 'numpy>=1.19.2' 'decorator>=4.4.0' 'sympy>=1.5.1' 'cffi>=1.12.3' 'protobuf>=3.13.0'
30+
31+
RUN pip3.8 install attrs pyyaml pathlib2 scipy requests psutil absl-py && \
32+
pip3.9 install attrs pyyaml pathlib2 scipy requests psutil absl-py && \
33+
pip3.10 install attrs pyyaml pathlib2 scipy requests psutil absl-py
34+
35+
# update envs for driver
36+
ENV LD_LIBRARY_PATH=/usr/local/Ascend/driver/lib64:$LD_LIBRARY_PATH
37+
ENV LD_LIBRARY_PATH=/usr/local/Ascend/driver/lib64/common:$LD_LIBRARY_PATH
38+
ENV LD_LIBRARY_PATH=/usr/local/Ascend/driver/lib64/driver:$LD_LIBRARY_PATH
39+
40+
# Install Ascend toolkit
41+
RUN wget -q --no-proxy https://paddle-ascend.bj.bcebos.com/cvmodel/ascend-materials/Ascend-cann-toolkit_${CANN_VERSION}_linux-${SYSTEM}.run --no-check-certificate && \
42+
chmod +x Ascend-cann-toolkit_${CANN_VERSION}_linux-*.run && \
43+
./Ascend-cann-toolkit_${CANN_VERSION}_linux-*.run --install --quiet && \
44+
echo "source /usr/local/Ascend/ascend-toolkit/set_env.sh" >>~/.bashrc && \
45+
rm -rf Ascend-cann-toolkit_${CANN_VERSION}_linux-*.run
46+
47+
# Install Ascend Kernels
48+
ARG ASCEND_KLS=Ascend-cann-kernels-${NPU_VERSION}_${CANN_VERSION}_linux-${SYSTEM}.run
49+
RUN wget -q --no-proxy https://paddle-ascend.bj.bcebos.com/cvmodel/ascend-materials/${ASCEND_KLS} --no-check-certificate && \
50+
chmod +x ${ASCEND_KLS} && ./${ASCEND_KLS} --install --quiet && rm -rf ${ASCEND_KLS}
51+
52+
# Install Ascend nnal
53+
RUN wget -q --no-proxy https://paddle-ascend.bj.bcebos.com/cvmodel/ascend-materials/Ascend-cann-nnal_${CANN_VERSION}_linux-${SYSTEM}.run --no-check-certificate && \
54+
chmod +x Ascend-cann-nnal_${CANN_VERSION}_linux-${SYSTEM}.run && \
55+
. /usr/local/Ascend/ascend-toolkit/set_env.sh && \
56+
./Ascend-cann-nnal_${CANN_VERSION}_linux-${SYSTEM}.run --install --quiet &&\
57+
rm -rf Ascend-cann-nnal_${CANN_VERSION}_linux-${SYSTEM}.run && \
58+
echo "source /usr/local/Ascend/nnal/atb/set_env.sh" >> /root/.bashrc
59+
60+
# install post process ops
61+
RUN wget -q --no-proxy https://paddle-ascend.bj.bcebos.com/code-share-master.zip --no-check-certificate && \
62+
. /usr/local/Ascend/ascend-toolkit/set_env.sh && \
63+
unzip code-share-master.zip && \
64+
cd code-share-master/build && bash build_ops.sh > /dev/null && \
65+
chmod +x aie_ops.run && ./aie_ops.run --extract=/usr/local/Ascend/ && \
66+
rm -rf /usr/local/Ascend/code-share-master
67+
68+
# update env for ascendc
69+
ENV ASCEND_CUSTOM_OPP_PATH=/usr/local/Ascend/vendors/aie_ascendc
70+
71+
# DEV image should open error level log
72+
# 0 debug; 1 info; 2 warning; 3 error; 4 null
73+
ENV ASCEND_GLOBAL_LOG_LEVEL=3
74+
75+
# environment for HCCL
76+
ENV HCCL_CONNECT_TIMEOUT=7200
77+
ENV HCCL_WHITELIST_DISABLE=1
78+
ENV HCCL_SECURITY_MODE=1
79+
ENV HCCL_BUFFSIZE=120
80+
81+
# environment for PaddlePaddle
82+
ENV FLAGS_npu_storage_format=0
83+
ENV FLAGS_use_stride_kernel=0
84+
ENV FLAGS_allocator_strategy=naive_best_fit
85+
ENV PADDLE_XCCL_BACKEND=npu
86+
87+
RUN rm -rf /usr/local/Ascend/driver
10488

10589
# Clean
10690
RUN apt-get clean -y

‎tools/dockerfile/build-image.sh

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
#!/bin/bash
2+
3+
# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
set -ex
18+
19+
# Usage:
20+
# export CANN_VERSION=8.0.RC1
21+
# bash build-image.sh ${CANN_VERSION}
22+
23+
CANN_VERSION=${1:-8.0.0} # default 8.0.0
24+
SYSTEM=${2:-x86_64} # default x86_64
25+
NPU_VERSION=${3:-910b} # default 910b
26+
27+
DOCKER_VERSION=${CANN_VERSION//[^0-9A-Z]/} # 80T13
28+
29+
# Download packages from https://www.hiascend.com/software/cann/community first
30+
if [ ! -f Ascend-cann-toolkit_${CANN_VERSION}_linux-$(uname -m).run ]; then
31+
echo "Please download CANN installation packages first!"
32+
exit 1
33+
fi
34+
35+
sed "s#<baseimg>#registry.baidubce.com/device/paddle-cpu:ubuntu20-npu-base-$(uname -m)-gcc84#g" Dockerfile.npu.ubuntu20.gcc84 > Dockerfile.npu.ubuntu20.gcc84.test
36+
docker build --network=host -f Dockerfile.npu.ubuntu20.gcc84.test \
37+
--build-arg CANN_VERSION=${CANN_VERSION} \
38+
--build-arg SYSTEM=${SYSTEM} \
39+
--build-arg NPU_VERSION=${NPU_VERSION} \
40+
--build-arg http_proxy=${proxy} \
41+
--build-arg https_proxy=${proxy} \
42+
--build-arg ftp_proxy=${proxy} \
43+
--build-arg no_proxy=bcebos.com \
44+
-t ccr-2vdh3abv-pub.cnc.bj.baidubce.com/device/paddle-npu:cann${DOCKER_VERSION}-ubuntu20-npu-${NPU_VERSION}-base-$(uname -m)-gcc84 .
45+
docker push ccr-2vdh3abv-pub.cnc.bj.baidubce.com/device/paddle-npu:cann${DOCKER_VERSION}-ubuntu20-npu-${NPU_VERSION}-base-$(uname -m)-gcc84
46+
rm -rf Dockerfile.npu.ubuntu20.gcc84.test

‎tools/dockerfile/manylinux/Dockerfile-118

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# A image for building paddle binaries
2+
# Use cuda devel base image for both cpu and gpu environment
3+
# When you modify it, please be aware of cudnn-runtime version
4+
ARG CUDA_VERSION=11.8
5+
ARG CUDA_PADDLE=cu118
6+
ARG BASE_TARGET=cuda${CUDA_VERSION}
7+
8+
FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 as base
9+
MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
10+
11+
12+
# ENV variables
13+
ARG WITH_GPU
14+
ARG WITH_AVX
15+
ARG PYTHON_VERSION=3.10
16+
17+
ENV WITH_GPU=${WITH_GPU:-ON}
18+
ENV WITH_AVX=${WITH_AVX:-ON}
19+
ENV DEBIAN_FRONTEND=noninteractive
20+
ENV LD_LIBRARY_PATH=/usr/local/cuda-${CUDA_VERSION}/targets/x86_64-linux/lib:$LD_LIBRARY_PATH
21+
22+
ENV HOME /root
23+
24+
RUN apt-get update --allow-unauthenticated && \
25+
apt-get install -y --no-install-recommends \
26+
git \
27+
vim \
28+
curl \
29+
wget \
30+
make \
31+
libssl-dev \
32+
autoconf \
33+
automake \
34+
libtool \
35+
python${PYTHON_VERSION} \
36+
python${PYTHON_VERSION}-dev \
37+
python3-pip && \
38+
ln -sf /usr/bin/python3 /usr/bin/python && \
39+
rm -rf /var/lib/apt/lists/*
40+
41+
WORKDIR /home
42+
RUN wget -q https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.tar.gz && \
43+
tar -zxf cmake-3.18.0-Linux-x86_64.tar.gz && \
44+
rm cmake-3.18.0-Linux-x86_64.tar.gz && \
45+
rm -rf /home/cmake-3.18.0-Linux-x86_64/doc /home/cmake-3.18.0-Linux-x86_64/man
46+
47+
ENV PATH=/home/cmake-3.18.0-Linux-x86_64/bin:$PATH
48+
49+
50+
ARG TMP_DIR=patchelf_tmp
51+
RUN rm -rf "$TMP_DIR" && git clone -b 0.15.0 https://github.com/NixOS/patchelf "$TMP_DIR" && \
52+
cd "$TMP_DIR" && ./bootstrap.sh && \
53+
./configure && make && make install && \
54+
cd .. && rm -rf "$TMP_DIR"
55+
56+
RUN wget -q https://paddle-ci.gz.bcebos.com/ccache-4.8.2.tar.gz && \
57+
tar xf ccache-4.8.2.tar.gz && mkdir /usr/local/ccache-4.8.2 && cd ccache-4.8.2 && \
58+
mkdir build && cd build && \
59+
cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.8.2 .. && \
60+
make -j8 && make install && \
61+
ln -s /usr/local/ccache-4.8.2/bin/ccache /usr/local/bin/ccache && \
62+
cd ../../ && rm -rf ccache-4.8.2.tar.gz && rm -rf ccache-4.8.2
63+
64+
RUN python -m pip install --no-cache-dir paddlepaddle-gpu==3.0.0 -i https://www.paddlepaddle.org.cn/packages/stable/cu118/

‎tools/dockerfile/manylinux/Dockerfile-126

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# A image for building paddle binaries
2+
# Use cuda devel base image for both cpu and gpu environment
3+
# When you modify it, please be aware of cudnn-runtime version
4+
ARG CUDA_VERSION=12.6
5+
ARG CUDA_PADDLE=126
6+
ARG BASE_TARGET=cuda${CUDA_VERSION}
7+
8+
FROM nvidia/cuda:12.6.0-cudnn-devel-ubuntu22.04 as base
9+
#FROM nvidia/cuda:12.6.0-devel-ubuntu22.04 as base
10+
MAINTAINER PaddlePaddle Authors <paddle-dev@baidu.com>
11+
12+
13+
# ENV variables
14+
ARG WITH_GPU
15+
ARG WITH_AVX
16+
ARG PYTHON_VERSION=3.10
17+
18+
ENV WITH_GPU=${WITH_GPU:-ON}
19+
ENV WITH_AVX=${WITH_AVX:-ON}
20+
ENV DEBIAN_FRONTEND=noninteractive
21+
ENV LD_LIBRARY_PATH=/usr/local/cuda-${CUDA_VERSION}/targets/x86_64-linux/lib:$LD_LIBRARY_PATH
22+
23+
ENV HOME /root
24+
25+
RUN apt-get update --allow-unauthenticated && \
26+
apt-get install -y --no-install-recommends \
27+
git \
28+
vim \
29+
curl \
30+
wget \
31+
make \
32+
libssl-dev \
33+
autoconf \
34+
automake \
35+
libtool \
36+
python${PYTHON_VERSION} \
37+
python${PYTHON_VERSION}-dev \
38+
python3-pip && \
39+
ln -sf /usr/bin/python3 /usr/bin/python && \
40+
rm -rf /var/lib/apt/lists/*
41+
42+
WORKDIR /home
43+
RUN wget -q https://cmake.org/files/v3.18/cmake-3.18.0-Linux-x86_64.tar.gz && \
44+
tar -zxf cmake-3.18.0-Linux-x86_64.tar.gz && \
45+
rm cmake-3.18.0-Linux-x86_64.tar.gz && \
46+
rm -rf /home/cmake-3.18.0-Linux-x86_64/doc /home/cmake-3.18.0-Linux-x86_64/man
47+
48+
ENV PATH=/home/cmake-3.18.0-Linux-x86_64/bin:$PATH
49+
50+
51+
ARG TMP_DIR=patchelf_tmp
52+
RUN rm -rf "$TMP_DIR" && git clone -b 0.15.0 https://github.com/NixOS/patchelf "$TMP_DIR" && \
53+
cd "$TMP_DIR" && ./bootstrap.sh && \
54+
./configure && make && make install && \
55+
cd .. && rm -rf "$TMP_DIR"
56+
57+
RUN wget -q https://paddle-ci.gz.bcebos.com/ccache-4.8.2.tar.gz && \
58+
tar xf ccache-4.8.2.tar.gz && mkdir /usr/local/ccache-4.8.2 && cd ccache-4.8.2 && \
59+
mkdir build && cd build && \
60+
cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local/ccache-4.8.2 .. && \
61+
make -j8 && make install && \
62+
ln -s /usr/local/ccache-4.8.2/bin/ccache /usr/local/bin/ccache && \
63+
cd ../../ && rm -rf ccache-4.8.2.tar.gz && rm -rf ccache-4.8.2
64+
65+
RUN python -m pip install --no-cache-dir paddlepaddle-gpu==3.0.0 -i https://www.paddlepaddle.org.cn/packages/stable/cu126/

0 commit comments

Comments
 (0)