create-env

#!/bin/bash

#######################################################################
#
# versions in this release

gfal2_bindings_version=1.12.2
gfal2_util_version=1.8.1
rucio_version=32.8.0
## REMEMBER THE 'v' IN CUTAX_VERSION!! (unless it is 'latest')
cutax_version=v2.1.2
nestpy_version=v2.0.4
#######################################################################

set -e

target_dir=$1

xenonnt_tag=$2
if [ "x$xenonnt_tag" = "x" ]; then
    xenonnt_tag=development
fi

env_name=XENONnT_${xenonnt_tag}

if [ "X$target_dir" = "X" ]; then
    echo "Please specify a target directory. Example: ./create-env /tmp/myenv" >&1
    exit 1
fi

if [ -e $target_dir ]; then
    echo "Target directory already exists - refusing to work on it" >&1
    exit 1
fi
mkdir -p $target_dir


function announce {
    echo
    echo "#######################################################################################"
    echo "## $1       ("`date -u`")"
    echo
}


function run_quietly {
    log=`mktemp --suffix=.log`
    rc=0
    echo "Running quietly: $@"
    if ! $@ >$log 2>&1; then
        rc=$?
        tail -n 500 $log
    fi
    rm -f $log
    return $rc
}


# build environment
export LC_ALL=en_US.utf8
export LANG=en_US.utf8

announce "Installing Miniconda"
wget -nv -O conda_setup.sh https://repo.anaconda.com/miniconda/Miniconda3-py311_24.9.2-0-Linux-x86_64.sh

bash conda_setup.sh -b -p $target_dir/anaconda
rm -f conda_setup.sh


export PKG_CONFIG_PATH=${target_dir}/anaconda/envs/${env_name}/lib64/pkgconfig:/usr/lib64/pkgconfig:/usr/share/pkgconfig

announce "Updating conda"
$target_dir/anaconda/bin/conda update conda --yes

announce "Installing Anaconda environment"
echo -e "name: ${env_name}\n$(cat conda_xnt.yml | grep -v '^name:.*')" > conda_xnt_tmp.yml         # Add name to the env yml
$target_dir/anaconda/bin/conda env create -f conda_xnt_tmp.yml

announce "Cleaning Anaconda environment"
$target_dir/anaconda/bin/conda clean --all --yes
rm -f ${target_dir}/anaconda/pkgs/rope-0.11.0-py37_0/info/LICENSE.txt

announce "Activating Anaconda environment"
source $target_dir/anaconda/bin/activate ${env_name}

# Forging nodejs (for jupyter)
announce "Installing nodejs"
conda install -c conda-forge nodejs=18.15.0 --yes

# Install to avoid md2 problem in openssl
conda install anaconda::openldap --yes

# boost
announce "Installing Boost"
cd ${target_dir}
rm -rf boost_*
wget -nv https://archives.boost.io/release/1.87.0/source/boost_1_87_0.tar.bz2
tar xjf boost_1_87_0.tar.bz2
rm -f boost_1_87_0.tar.bz2
cd boost_1_87_0
./bootstrap.sh \
    --prefix=${target_dir}/anaconda/envs/${env_name} --show-libraries \
    --with-libraries=atomic,chrono,date_time,exception,filesystem,graph,iostreams,locale,log,math,program_options,random,regex,serialization,system,test,thread,timer,wave,python \
    --with-python=${target_dir}/anaconda/envs/${env_name}/bin/python3
./b2 install --prefix=${target_dir}/anaconda/envs/${env_name}
cd ${target_dir}
rm -rf boost_*
# required by gfal2-python, but maybe others
ln -s ${target_dir}/anaconda/envs/${env_name}/lib/libboost_python311.so ${target_dir}/anaconda/envs/${env_name}/lib/libboost_python.so
ln -s ${target_dir}/anaconda/envs/${env_name}/lib/libboost_python311.a ${target_dir}/anaconda/envs/${env_name}/lib/libboost_python.a

# gfal2-python bindings
announce "Installing GFAL2 Python bindings"
git clone https://gitlab.cern.ch/dmc/gfal2-bindings.git
cd gfal2-bindings
git checkout v$gfal2_bindings_version
perl -p -i -e "s;.*\\\${Boost_LIBRARYDIR};            \"${target_dir}/anaconda/envs/${env_name}/lib\";" CMakeLists.txt
cmake -DPYTHON_EXECUTABLE=${target_dir}/anaconda/envs/${env_name}/bin/python3.11 \
    -DPYTHON_EXECUTABLE_3=${target_dir}/anaconda/envs/${env_name}/bin/python3.11 \
    -DPYTHON_EXECUTABLE_3.11=${target_dir}/anaconda/envs/${env_name}/bin/python3.11 \
    -DBOOST_ROOT=${target_dir}/anaconda/envs/${env_name} \
    -DSKIP_DOC=TRUE -DSKIP_TESTS=TRUE
make
make install
cd ${target_dir}
rm -rf gfal2-bindings

# gfal2 clients
announce "Installing GFAL2 clients"
git clone https://gitlab.cern.ch/dmc/gfal2-util.git
cd gfal2-util/
git checkout v$gfal2_util_version
python3 setup.py install --prefix=${target_dir}/anaconda/envs/${env_name}
cd ${target_dir}
rm -rf gfal2-util

# rucio-clients
announce "Installing Rucio"
git clone https://github.com/rucio/rucio.git
cd rucio
# For some strange reason `git checkout $rucio_version` does not work 2022/07/22
git checkout $rucio_version -f
# don't mess with our version of setuptools, urllib3 and requests
perl -p -i -e 's/^setuptools.*//' etc/pip-requires-client
perl -p -i -e 's/^urllib3.*//' etc/pip-requires-client
perl -p -i -e 's/^requests.*//' etc/pip-requires-client
perl -p -i -e 's/six>=1.12.0<1.15.0/six>=1.12.0,<1.15.0/' etc/pip-requires-client
python3 setup_rucio_client.py install --prefix=${target_dir}/anaconda/envs/${env_name}
cd ${target_dir}
rm -rf rucio

# rucio config
cat >${target_dir}/anaconda/envs/${env_name}/etc/rucio.cfg <<EOF
[common]
logdir = /var/log/rucio
loglevel = DEBUG
mailtemplatedir=/opt/rucio/etc/mail_templates

[client]
rucio_host = https://rucio-xenon.grid.uchicago.edu:443
auth_host = https://rucio-xenon.grid.uchicago.edu:443
auth_type = x509_proxy

# ca_cert = /etc/grid-security/certificates
ca_cert = \$X509_CERT_DIR
client_cert = \$X509_USER_PROXY
client_key = \$X509_USER_PROXY
client_x509_proxy = \$X509_USER_PROXY
request_retries = 3
EOF

# nestpy
announce "Installing nestpy"
git clone --branch $nestpy_version --single-branch https://github.com/NESTCollaboration/nestpy.git
cd nestpy
git submodule update --init --recursive
pip install .
cd ${target_dir}
rm -rf nestpy

announce "Adding setup.sh"

# generate setup.sh file for easy sourcing
cat >${target_dir}/setup.sh <<EOF
#!/bin/bash

for VAR in X509_CERT_DIR X509_VOMS_DIR; do
    VALUE=\${!VAR}
    if [ "X\$VALUE" != "X" ]; then
        echo "WARNING: \$VAR is set and could lead to problems when using this environment" 1>&2
    fi
done

# Need a clean environment - this is to prevent inheriting host specifics from the outside
# Note that "unset" is not great here, as Singularity might just take that as
# keeping the value from the host instead of clearing it.
export OSG_LOCATION=""
export GLOBUS_LOCATION=""
export PYTHONPATH=""
export PERL5LIB=""

# when inside a container, reset some variables from outside

if [ "x\$SINGULARITY_CONTAINER" != "x" ]; then
    export PATH=/usr/bin:/usr/local/bin
    export LD_LIBRARY_PATH=""
# if not in a singularity container, use cvmfs CA certificates
else
    # first check if the directory is mounted
    if [ -e "/cvmfs/oasis.opensciencegrid.org/mis/osg-wn-client/certificates" ]; then
        export X509_CERT_DIR=/cvmfs/oasis.opensciencegrid.org/mis/osg-wn-client/certificates
    fi
fi

# devtools-9 environment for updated compilers
if [ -e /opt/rh/devtoolset-9/enable ]; then
    source /opt/rh/devtoolset-9/enable
fi

source ${target_dir}/anaconda/bin/activate ${env_name}

# prepend to LD_LIBRARY_PATH - non-Python tools might be using it
export LD_LIBRARY_PATH=\$CONDA_PREFIX/lib64:\$CONDA_PREFIX/lib\${LD_LIBRARY_PATH:+:}\${LD_LIBRARY_PATH}

# set GFAL_CONFIG_DIR and GFAL_PLUGIN_DIR explicitly so that we do not pick up a bad values from the site
export GFAL_CONFIG_DIR="/etc/gfal2.d"
export GFAL_PLUGIN_DIR="/usr/lib64/gfal2-plugins"

# rucio
export RUCIO_HOME=\$CONDA_PREFIX
export RUCIO_ACCOUNT=xenon-analysis
if [ "x\$X509_CERT_DIR" = "x" ]; then
    export X509_CERT_DIR=/etc/grid-security/certificates
fi


# site-specific stuff
for space in /project/lgrandi/grid_proxy /project2/lgrandi/grid_proxy /xenon/grid_proxy; do
    if [ -d \$space ]; then
        if [ "x\$X509_USER_PROXY" = "x" ]; then
            export X509_USER_PROXY=\${space}/xenon_service_proxy
        fi
        # change where we cache the .pyc files if we are on one of these sites.
        export PYTHONPYCACHEPREFIX=/tmp/\$USER
    fi
done

# xenon config location on midway
config_path_midway2=/project2/lgrandi/xenonnt/xenon.config
config_path_midway3=/project/lgrandi/xenonnt/xenon.config
if [ -e \$config_path_midway2 ]; then
    export XENON_CONFIG=\$config_path_midway2
elif [ -e \$config_path_midway3 ]; then
    export XENON_CONFIG=\$config_path_midway3
fi

# grab a local cutax installation if we have one
DALI_CUTAX_DIR=/dali/lgrandi/xenonnt/software/cutax/${cutax_version}
MIDWAY2_CUTAX_DIR=/project2/lgrandi/xenonnt/software/cutax/${cutax_version}
MIDWAY3_CUTAX_DIR=/project/lgrandi/xenonnt/software/cutax/${cutax_version}
OSG_CUTAX_DIR=/ospool/uc-shared/project/xenon/xenonnt/software/cutax/${cutax_version}

if [ "x\${CUTAX_LOCATION}" = "x" ]; then
    for dir in \${DALI_CUTAX_DIR} \${MIDWAY2_CUTAX_DIR} \${MIDWAY3_CUTAX_DIR} \${OSG_CUTAX_DIR}; do
        if [ -e \$dir ]; then
            CUTAX_LOCATION=\$dir
        fi
    done
fi

if [ "x\$INSTALL_CUTAX" = "x" ]; then
    export INSTALL_CUTAX=1
fi

if [ \$INSTALL_CUTAX -eq 1 ]; then
    if [ -e "\${CUTAX_LOCATION}" ]; then
        export CUTAX_LOCATION
        export PYTHONPATH=\${CUTAX_LOCATION}:\$PYTHONPATH
    fi
fi

# set env variable for matplotlib styles on Midway
MPL_DIR='/dali/lgrandi/xenonnt/software/nton/mplconfigs/'
if [ -e \$MPL_DIR ]; then
    export MPLCONFIGDIR=\$MPL_DIR
fi

EOF

announce "Adding zzz-50-xenon.sh"

# this will be sourced when entering the container, but
# should be sourced last in order to pick up for example
# SINGULARITYENV_... vars
mkdir -p /.singularity.d/env
cat > /.singularity.d/env/zzz-50-xenon.sh <<EOF
#!/bin/sh

if [ "x\$XENON_DEBUG" != "x" ]; then
    echo 1>&2
    echo "Environment before sourcing XENONnT setup:" 1>&2
    (env | sort) 1>&2
    echo 1>&2
fi

if [ "x\$XENON_SOURCE" = "x" ]; then
    export XENON_SOURCE=1
fi

if [ \$XENON_SOURCE -eq 1 ]; then
    source ${target_dir}/setup.sh
fi

if [ "x\$XENON_DEBUG" != "x" ]; then
    echo 1>&2
    echo "Environment after sourcing XENONnT setup:" 1>&2
    (env | sort) 1>&2
    echo 1>&2
fi
EOF

# Done testing
announce "All done!"