Skip to content

Commit 431a736

Browse files
thxCodethxCode
committed
ci: refactor build
Signed-off-by: thxCode <[email protected]>
1 parent 8f76f93 commit 431a736

File tree

12 files changed

+1019
-24
lines changed

12 files changed

+1019
-24
lines changed

‎.github/workflows/build-devops-images.yml

+342-24
Large diffs are not rendered by default.

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
.DS_Store

‎cann/cann.sh

+141
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
get_architecture() {
6+
# not case sensitive
7+
shopt -s nocasematch
8+
9+
case "${PLATFORM}" in
10+
"linux/x86_64"|"linux/amd64")
11+
ARCH="x86_64"
12+
;;
13+
"linux/aarch64"|"linux/arm64")
14+
ARCH="aarch64"
15+
;;
16+
*)
17+
echo "Error: Unsupported architecture ${PLATFORM}."
18+
exit 1
19+
;;
20+
esac
21+
22+
echo "${ARCH}"
23+
}
24+
25+
download_file() {
26+
set +e
27+
28+
local max_retries=10
29+
local retry_delay=10
30+
local url="$1"
31+
local path="$2"
32+
33+
for ((i=1; i<=max_retries; i++)); do
34+
echo "Attempt $i of $max_retries..."
35+
curl -fsSL -o "${path}" "${url}"
36+
if [[ $? -eq 0 ]]; then
37+
return 0
38+
else
39+
echo "Download failed with error code $?. Retrying in ${retry_delay} seconds..."
40+
sleep ${retry_delay}
41+
fi
42+
done
43+
44+
echo "All attempts failed. Exiting."
45+
return 1
46+
}
47+
48+
download_cann() {
49+
if [[ ${CANN_VERSION} == "8.0.RC2.alpha001" ]]; then
50+
local url_prefix="https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/Milan-ASL/Milan-ASL%20V100R001C18B800TP015"
51+
elif [[ ${CANN_VERSION} == "8.0.RC2.alpha002" ]]; then
52+
local url_prefix="https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/Milan-ASL/Milan-ASL%20V100R001C18SPC805"
53+
elif [[ ${CANN_VERSION} == "8.0.RC2.alpha003" ]]; then
54+
local url_prefix="https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/Milan-ASL/Milan-ASL%20V100R001C18SPC703"
55+
else
56+
local url_prefix="https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/CANN/CANN%20${CANN_VERSION}"
57+
fi
58+
local url_suffix="response-content-type=application/octet-stream"
59+
local toolkit_url="${url_prefix}/${TOOLKIT_FILE}?${url_suffix}"
60+
local kernels_url="${url_prefix}/${KERNELS_FILE}?${url_suffix}"
61+
62+
if [ ! -f "${TOOLKIT_PATH}" ]; then
63+
echo "Downloading ${TOOLKIT_FILE} from ${toolkit_url}"
64+
download_file "${toolkit_url}" "${TOOLKIT_PATH}"
65+
fi
66+
67+
if [ ! -f "${KERNELS_PATH}" ]; then
68+
echo "Downloading ${KERNELS_FILE} from ${kernels_url}"
69+
download_file "${kernels_url}" "${KERNELS_PATH}"
70+
fi
71+
72+
echo "CANN ${CANN_VERSION} download successful."
73+
}
74+
75+
set_env() {
76+
local cann_toolkit_env_file="${CANN_HOME}/ascend-toolkit/set_env.sh"
77+
if [ ! -f "${cann_toolkit_env_file}" ]; then
78+
echo "CANN Toolkit ${CANN_VERSION} installation failed."
79+
exit 1
80+
else
81+
local driver_path_env="LD_LIBRARY_PATH=${CANN_HOME}/driver/lib64/common/:${CANN_HOME}/driver/lib64/driver/:\${LD_LIBRARY_PATH}" && \
82+
echo "export ${driver_path_env}" >> /etc/profile
83+
echo "export ${driver_path_env}" >> ~/.bashrc
84+
echo "source ${cann_toolkit_env_file}" >> /etc/profile
85+
echo "source ${cann_toolkit_env_file}" >> ~/.bashrc
86+
source ${cann_toolkit_env_file}
87+
fi
88+
}
89+
90+
install_cann() {
91+
# Download installers
92+
if [ ! -f "${TOOLKIT_PATH}" ] || [ ! -f "${KERNELS_PATH}" ]; then
93+
echo "[WARNING] Installers do not exist, re-download them."
94+
download_cann
95+
fi
96+
97+
# Install dependencies
98+
pip install --no-cache-dir --upgrade pip
99+
pip install --no-cache-dir \
100+
attrs cython numpy decorator sympy cffi pyyaml pathlib2 psutil protobuf scipy requests absl-py
101+
102+
# Install CANN Toolkit
103+
echo "Installing ${TOOLKIT_FILE}"
104+
chmod +x "${TOOLKIT_PATH}"
105+
bash "${TOOLKIT_PATH}" --quiet --install --install-for-all --install-path="${CANN_HOME}"
106+
rm -f "${TOOLKIT_PATH}"
107+
108+
# Set environment variables
109+
set_env
110+
111+
# Install CANN Kernels
112+
echo "Installing ${KERNELS_FILE}"
113+
chmod +x "${KERNELS_PATH}"
114+
bash "${KERNELS_PATH}" --quiet --install --install-for-all --install-path="${CANN_HOME}"
115+
rm -f "${KERNELS_PATH}"
116+
117+
echo "CANN ${CANN_VERSION} installation successful."
118+
}
119+
120+
PLATFORM=${PLATFORM:=$(uname -s)/$(uname -m)}
121+
ARCH=$(get_architecture)
122+
CANN_HOME=${CANN_HOME:="/usr/local/Ascend"}
123+
CANN_CHIP=${CANN_CHIP:="910b"}
124+
CANN_VERSION=${CANN_VERSION:="8.0.RC1"}
125+
126+
TOOLKIT_FILE="Ascend-cann-toolkit_${CANN_VERSION}_linux-${ARCH}.run"
127+
KERNELS_FILE="Ascend-cann-kernels-${CANN_CHIP}_${CANN_VERSION}_linux.run"
128+
TOOLKIT_PATH="/tmp/${TOOLKIT_FILE}"
129+
KERNELS_PATH="/tmp/${KERNELS_FILE}"
130+
131+
# Parse arguments
132+
if [ "$1" == "--download" ]; then
133+
download_cann
134+
elif [ "$1" == "--install" ]; then
135+
install_cann
136+
elif [ "$1" == "--set_env" ]; then
137+
set_env
138+
else
139+
echo "Unexpected arguments, use '--download', '--install' or '--set_env' instead"
140+
exit 1
141+
fi

‎cann/ubuntu/Dockerfile

+100
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
ARG CANN_VERSION=8.0.rc2.alpha003
2+
ARG CANN_CHIP=910b
3+
ARG BASE_VERSION=20.04
4+
5+
#
6+
# PREPARE
7+
#
8+
9+
FROM ubuntu:${BASE_VERSION} as cann-installer
10+
RUN apt-get update -y \
11+
&& apt-get install -y \
12+
apt-transport-https \
13+
ca-certificates \
14+
bash \
15+
git \
16+
wget \
17+
gcc \
18+
g++ \
19+
make \
20+
cmake \
21+
zlib1g \
22+
zlib1g-dev \
23+
openssl \
24+
libsqlite3-dev \
25+
libssl-dev \
26+
libffi-dev \
27+
unzip \
28+
pciutils \
29+
net-tools \
30+
libblas-dev \
31+
gfortran \
32+
patchelf \
33+
libblas3 \
34+
&& apt-get clean \
35+
&& rm -rf /var/lib/apt/lists/* \
36+
&& rm -rf /var/tmp/* \
37+
&& rm -rf /tmp/*
38+
COPY cann.sh /tmp/cann.sh
39+
ENV CANN_VERSION=${CANN_VERSION} \
40+
CANN_CHIP=${CANN_CHIP}
41+
RUN bash /tmp/cann.sh --download
42+
RUN bash /tmp/cann.sh --install
43+
44+
#
45+
# MAIN
46+
#
47+
48+
FROM ubuntu:${BASE_VERSION}
49+
ENV LD_LIBRARY_PATH="/usr/local/Ascend/driver/lib64/common/:/usr/local/Ascend/driver/lib64/driver/:$LD_LIBRARY_PATH"
50+
COPY --from=cann-installer /usr/local/Ascend /usr/local/Ascend
51+
COPY --from=cann-installer /etc/Ascend /etc/Ascend
52+
53+
# SOURCE
54+
RUN apt-get update -y \
55+
&& apt-get install -y software-properties-common \
56+
&& add-apt-repository -y ppa:ubuntu-toolchain-r/test \
57+
&& apt-get update -y
58+
59+
# MAKE & CCACHE & CURL & GIT
60+
RUN apt-get install -y make ccache curl git \
61+
&& make --version \
62+
&& ccache --version \
63+
&& curl --version \
64+
&& git --version
65+
66+
# CMAKE
67+
ENV CMAKE_VERSION=3.22.1
68+
RUN curl -sL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz" | tar -zx -C /usr --strip-components 1 \
69+
&& cmake --version
70+
71+
# GCC
72+
RUN apt-get install -y binutils pkg-config gcc-11 g++-11 \
73+
&& update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 10 \
74+
&& update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 10 \
75+
&& ldconfig -v \
76+
&& gcc --version
77+
RUN apt-get install -y build-essential
78+
79+
# OPENSSL
80+
ENV OPENSSL_VESION=3.4.0 \
81+
PKG_CONFIG_PATH="/usr/local/openssl/lib/pkgconfig:/usr/local/openssl/lib64/pkgconfig:$PKG_CONFIG_PATH" \
82+
LD_LIBRARY_PATH="/usr/local/openssl/lib:/usr/local/openssl/lib64:$LD_LIBRARY_PATH"
83+
RUN apt-get install -y zlib1g-dev perl \
84+
&& curl -sL "https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VESION}/openssl-${OPENSSL_VESION}.tar.gz" | tar -zx -C /tmp \
85+
&& loc=$(pwd) && cd /tmp/openssl-${OPENSSL_VESION} \
86+
&& ./config --prefix=/usr/local/openssl --openssldir=/etc/ssl shared zlib enable-ssl3 enable-ssl3-method enable-mdc2 enable-md2 \
87+
&& make -j $(nproc) build_sw && make -j $(nproc) install_sw \
88+
&& cd ${local} && rm -rf /tmp/openssl-${OPENSSL_VESION} \
89+
&& ln -vsf /usr/local/openssl/bin/* /usr/bin/ \
90+
&& ln -vsf /usr/local/openssl/include/openssl /usr/include/openssl \
91+
&& echo "/usr/local/openssl/lib" >> /etc/ld.so.conf \
92+
&& echo "/usr/local/openssl/lib64" >> /etc/ld.so.conf \
93+
&& ldconfig -v \
94+
&& openssl --version
95+
96+
# OPENBLAS
97+
RUN apt-get install -y libopenblas-dev
98+
99+
# OPENMP
100+
RUN apt-get install -y libgomp1 libomp-dev

‎cpu/centos/Dockerfile

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
ARG BASE_VERSION=7
2+
3+
4+
FROM centos:${BASE_VERSION}
5+
6+
# SOURCE
7+
RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo \
8+
&& sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo \
9+
&& sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
10+
RUN yum install -y centos-release-scl \
11+
&& sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo \
12+
&& sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo \
13+
&& sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo \
14+
&& yum update -y
15+
RUN yum install -y epel-release --enablerepo=extras \
16+
&& yum install -y https://packages.endpointdev.com/rhel/7/os/$(uname -m)/endpoint-repo.$(uname -m).rpm \
17+
&& yum update -y
18+
19+
# MAKE & CCACHE & CURL & GIT
20+
RUN yum install -y make ccache curl git \
21+
&& make --version \
22+
&& ccache --version \
23+
&& curl --version \
24+
&& git --version
25+
26+
# CMAKE
27+
ENV CMAKE_VERSION=3.22.1
28+
RUN curl -sL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz" | tar -zx -C /usr --strip-components 1 \
29+
&& cmake --version
30+
31+
# GCC
32+
ENV PKG_CONFIG_PATH="/opt/rh/devtoolset-9/root/usr/lib/pkgconfig:/opt/rh/devtoolset-9/root/usr/lib64/pkgconfig:$PKG_CONFIG_PATH" \
33+
LD_LIBRARY_PATH="/opt/rh/devtoolset-9/root/usr/lib:/opt/rh/devtoolset-9/root/usr/lib/dyninst:/opt/rh/devtoolset-9/root/usr/lib64:/opt/rh/devtoolset-9/root/usr/lib64/dyninst:$LD_LIBRARY_PATH"
34+
RUN yum install -y devtoolset-9 devtoolset-9-libatomic-devel devtoolset-9-elfutils-libelf-devel \
35+
&& ln -vsf /opt/rh/devtoolset-9/root/bin/* /usr/bin/ \
36+
&& echo "/opt/rh/devtoolset-9/root/usr/lib" >> /etc/ld.so.conf \
37+
&& echo "/opt/rh/devtoolset-9/root/usr/lib64" >> /etc/ld.so.conf \
38+
&& ldconfig -v \
39+
&& gcc --version
40+
RUN yum install -y glibc-static libstdc++-static
41+
42+
# OPENSSL
43+
ENV OPENSSL_VESION=3.4.0 \
44+
PKG_CONFIG_PATH="/usr/local/openssl/lib/pkgconfig:/usr/local/openssl/lib64/pkgconfig:$PKG_CONFIG_PATH" \
45+
LD_LIBRARY_PATH="/usr/local/openssl/lib:/usr/local/openssl/lib64:$LD_LIBRARY_PATH"
46+
RUN yum install -y zlib-devel perl perl-IPC-Cmd perl-Test-Simple perl-CPAN \
47+
&& curl -sL "https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VESION}/openssl-${OPENSSL_VESION}.tar.gz" | tar -zx -C /tmp \
48+
&& loc=$(pwd) && cd /tmp/openssl-${OPENSSL_VESION} \
49+
&& ./config --prefix=/usr/local/openssl --openssldir=/etc/ssl shared zlib enable-ssl3 enable-ssl3-method enable-mdc2 enable-md2 \
50+
&& make -j $(nproc) build_sw && make -j $(nproc) install_sw \
51+
&& cd ${local} && rm -rf /tmp/openssl-${OPENSSL_VESION} \
52+
&& ln -vsf /usr/local/openssl/bin/* /usr/bin/ \
53+
&& ln -vsf /usr/local/openssl/include/openssl /usr/include/openssl \
54+
&& echo "/usr/local/openssl/lib" >> /etc/ld.so.conf \
55+
&& echo "/usr/local/openssl/lib64" >> /etc/ld.so.conf \
56+
&& ldconfig -v \
57+
&& openssl --version
58+
59+
# OPENBLAS
60+
RUN yum install -y openblas-static
61+
COPY <<EOF /usr/lib64/pkgconfig/openblas.pc
62+
Name: OpenBLAS
63+
Description: OpenBLAS library
64+
Version: 0.3.3
65+
Libs: -L/usr/lib64 -lopenblas
66+
Cflags: -I/usr/include/openblas
67+
EOF
68+
69+
# OPENMP
70+
RUN yum install -y libgomp openmpi openmpi-devel

‎cpu/rockylinux/Dockerfile

+55
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
ARG BASE_VERSION=8.9
2+
3+
4+
FROM rockylinux:${BASE_VERSION}
5+
6+
# SOURCE
7+
RUN dnf install -y epel-release \
8+
&& dnf config-manager --set-enabled powertools
9+
10+
# MAKE & CCACHE & CURL & GIT
11+
RUN yum install -y make ccache curl git \
12+
&& make --version \
13+
&& ccache --version \
14+
&& curl --version \
15+
&& git --version
16+
17+
# CMAKE
18+
ENV CMAKE_VERSION=3.22.1
19+
RUN curl -sL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz" | tar -zx -C /usr --strip-components 1 \
20+
&& cmake --version
21+
22+
# GCC
23+
ENV PKG_CONFIG_PATH="/opt/rh/gcc-toolset-11/root/usr/lib/pkgconfig:/opt/rh/gcc-toolset-11/root/usr/lib64/pkgconfig:$PKG_CONFIG_PATH" \
24+
LD_LIBRARY_PATH="/opt/rh/gcc-toolset-11/root/usr/lib:/opt/rh/gcc-toolset-11/root/usr/lib/dyninst:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib64/dyninst:$LD_LIBRARY_PATH"
25+
RUN dnf install -y binutils pkgconfig gcc gcc-c++ gcc-toolset-11 \
26+
&& ln -vsf /opt/rh/gcc-toolset-11/root/bin/* /usr/bin/ \
27+
&& echo "/opt/rh/gcc-toolset-11/root/usr/lib" >> /etc/ld.so.conf \
28+
&& echo "/opt/rh/gcc-toolset-11/root/usr/lib64" >> /etc/ld.so.conf \
29+
&& ldconfig -v \
30+
&& gcc --version
31+
RUN dnf install -y glibc-static libstdc++-static
32+
33+
# OPENSSL
34+
ENV OPENSSL_VESION=3.4.0 \
35+
PKG_CONFIG_PATH="/usr/local/openssl/lib/pkgconfig:/usr/local/openssl/lib64/pkgconfig:$PKG_CONFIG_PATH" \
36+
LD_LIBRARY_PATH="/usr/local/openssl/lib:/usr/local/openssl/lib64:$LD_LIBRARY_PATH"
37+
RUN dnf install -y zlib-devel perl \
38+
&& curl -sL "https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VESION}/openssl-${OPENSSL_VESION}.tar.gz" | tar -zx -C /tmp \
39+
&& loc=$(pwd) && cd /tmp/openssl-${OPENSSL_VESION} \
40+
&& ./config --prefix=/usr/local/openssl --openssldir=/etc/ssl shared zlib enable-ssl3 enable-ssl3-method enable-mdc2 enable-md2 \
41+
&& make -j $(nproc) build_sw && make -j $(nproc) install_sw \
42+
&& cd ${local} && rm -rf /tmp/openssl-${OPENSSL_VESION} \
43+
&& ln -vsf /usr/local/openssl/bin/* /usr/bin/ \
44+
&& ln -vsf /usr/local/openssl/include/openssl /usr/include/openssl \
45+
&& echo "/usr/local/openssl/lib" >> /etc/ld.so.conf \
46+
&& echo "/usr/local/openssl/lib64" >> /etc/ld.so.conf \
47+
&& ldconfig -v \
48+
&& openssl --version
49+
50+
# OPENBLAS
51+
RUN dnf install -y openblas-static
52+
53+
# OPENMP
54+
RUN yum install -y libgomp openmpi openmpi-devel
55+

0 commit comments

Comments
 (0)