From 6de5031dd15385ab296c9be83535ad7779c27b1e Mon Sep 17 00:00:00 2001 From: atalman Date: Tue, 8 Aug 2023 10:18:36 -0700 Subject: [PATCH] Decouple aarch64 ci setup and build --- aarch64_linux/aarch64_ci_build.sh | 43 ++---------------------------- aarch64_linux/aarch64_ci_setup.sh | 44 +++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 41 deletions(-) mode change 100755 => 100644 aarch64_linux/aarch64_ci_build.sh create mode 100644 aarch64_linux/aarch64_ci_setup.sh diff --git a/aarch64_linux/aarch64_ci_build.sh b/aarch64_linux/aarch64_ci_build.sh old mode 100755 new mode 100644 index 41843cead..c374359c2 --- a/aarch64_linux/aarch64_ci_build.sh +++ b/aarch64_linux/aarch64_ci_build.sh @@ -1,47 +1,8 @@ #!/bin/bash set -eux -o pipefail -# This script is used to prepare the Docker container for aarch64_ci_wheel_build.py python script -# as we need to install conda and setup the python version for the build. - -CONDA_PYTHON_EXE=/opt/conda/bin/python -CONDA_EXE=/opt/conda/bin/conda -PATH=/opt/conda/bin:$PATH -LD_LIBRARY_PATH=/opt/conda/lib:$LD_LIBRARY_PATH - -############################################################################### -# Install OS dependent packages -############################################################################### -yum -y install epel-release -yum -y install less zstd libgomp - -############################################################################### -# Install conda -# disable SSL_verify due to getting "Could not find a suitable TLS CA certificate bundle, invalid path" -# when using Python version, less than the conda latest -############################################################################### -echo 'Installing conda-forge' -curl -L -o /mambaforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh -chmod +x /mambaforge.sh -/mambaforge.sh -b -p /opt/conda -rm /mambaforge.sh -/opt/conda/bin/conda config --set ssl_verify False -/opt/conda/bin/conda install -y -c conda-forge python=${DESIRED_PYTHON} numpy pyyaml setuptools patchelf pygit2 openblas -python --version -conda --version - -############################################################################### -# Exec libglfortran.a hack -# -# libgfortran.a from quay.io/pypa/manylinux2014_aarch64 is not compiled with -fPIC. -# This causes __stack_chk_guard@@GLIBC_2.17 on pytorch build. To solve, get -# ubuntu's libgfortran.a which is compiled with -fPIC -############################################################################### -cd ~/ -curl -L -o ~/libgfortran-10-dev.deb http://ports.ubuntu.com/ubuntu-ports/pool/universe/g/gcc-10/libgfortran-10-dev_10.5.0-1ubuntu1_arm64.deb -ar x ~/libgfortran-10-dev.deb -tar --use-compress-program=unzstd -xvf data.tar.zst -C ~/ -cp -f ~/usr/lib/gcc/aarch64-linux-gnu/10/libgfortran.a /opt/rh/devtoolset-10/root/usr/lib/gcc/aarch64-redhat-linux/10/ +SCRIPTPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +source $SCRIPTPATH/aarch64_ci_setup.sh ############################################################################### # Run aarch64 builder python diff --git a/aarch64_linux/aarch64_ci_setup.sh b/aarch64_linux/aarch64_ci_setup.sh new file mode 100644 index 000000000..c7065056a --- /dev/null +++ b/aarch64_linux/aarch64_ci_setup.sh @@ -0,0 +1,44 @@ +#!/bin/bash +set -eux -o pipefail + +# This script is used to prepare the Docker container for aarch64_ci_wheel_build.py python script +# as we need to install conda and setup the python version for the build. + +CONDA_PYTHON_EXE=/opt/conda/bin/python +CONDA_EXE=/opt/conda/bin/conda +PATH=/opt/conda/bin:$PATH +LD_LIBRARY_PATH=/opt/conda/lib:$LD_LIBRARY_PATH + +############################################################################### +# Install OS dependent packages +############################################################################### +yum -y install epel-release +yum -y install less zstd libgomp + +############################################################################### +# Install conda +# disable SSL_verify due to getting "Could not find a suitable TLS CA certificate bundle, invalid path" +# when using Python version, less than the conda latest +############################################################################### +echo 'Installing conda-forge' +curl -L -o /mambaforge.sh https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-aarch64.sh +chmod +x /mambaforge.sh +/mambaforge.sh -b -p /opt/conda +rm /mambaforge.sh +/opt/conda/bin/conda config --set ssl_verify False +/opt/conda/bin/conda install -y -c conda-forge python=${DESIRED_PYTHON} numpy pyyaml setuptools patchelf pygit2 openblas +python --version +conda --version + +############################################################################### +# Exec libglfortran.a hack +# +# libgfortran.a from quay.io/pypa/manylinux2014_aarch64 is not compiled with -fPIC. +# This causes __stack_chk_guard@@GLIBC_2.17 on pytorch build. To solve, get +# ubuntu's libgfortran.a which is compiled with -fPIC +############################################################################### +cd ~/ +curl -L -o ~/libgfortran-10-dev.deb http://ports.ubuntu.com/ubuntu-ports/pool/universe/g/gcc-10/libgfortran-10-dev_10.5.0-1ubuntu1_arm64.deb +ar x ~/libgfortran-10-dev.deb +tar --use-compress-program=unzstd -xvf data.tar.zst -C ~/ +cp -f ~/usr/lib/gcc/aarch64-linux-gnu/10/libgfortran.a /opt/rh/devtoolset-10/root/usr/lib/gcc/aarch64-redhat-linux/10/