-
Notifications
You must be signed in to change notification settings - Fork 55
Installation on DelftBlue
https://doc.dhpc.tudelft.nl/delftblue/ (requires login)
(20.11.2023)
module load 2023r1-gcc11
module load openmpi/4.1.4
module load cmake/3.24.3
module load netcdf-fortran/4.6.0
module load fftw/3.3.10
git clone https://github.com/dalesteam/dales
cd dales
mkdir build
cd build
export SYST=gnu-fast
cmake .. -DUSE_FFTW=True
make -j 8
(assuming git clone above is already done)
cd dales
mkdir build-sp
cd build-sp
export SYST=gnu-fast
cmake .. -DUSE_FFTW=True -DFIELD_PRECISION=32 -DPOIS_PRECISION=32
make -j 8
(5.9.2022, works without FFTW library)
module unload gcc/10.2.0
module load 2022r2
module load openmpi/4.1.1
module load netcdf-fortran/4.5.3
module load cmake/3.21.4
git clone https://github.com/dalesteam/dales
cd dales
mkdir build
cd build
export SYST=gnu-fast
cmake ..
make -j 8
Adapt ntasks according to your case.
ntasks = nprocx * nprocy where nprocx and nprocy are given in the namelist.
#!/bin/bash
#SBATCH --job-name="dales"
#SBATCH --partition=compute
#SBATCH --account=research-ceg-grs
#SBATCH --time=01:00:00
#SBATCH --ntasks=16
#SBATCH --cpus-per-task=1
#SBATCH --mem-per-cpu=1G
NAMOPTIONS=namoptions.001
module unload gcc/10.2.0
module load 2022r2
module load openmpi/4.1.1
module load netcdf-fortran/4.5.3
DALES=$HOME/dales/build/src/dales4.4
srun $DALES $NAMOPTIONS
-
HYPRE is not yet available in the 2022r2 module set (except for the intel compiler)
-
With the intel compiler, netCDF (and FFTW) are not available.
When using FFTW:
- gfortran 8.5.0 reports an error on contiguous, issue: https://github.com/dalesteam/dales/issues/78
-
module load fftw3/openmpi/gcc/64/3.3.8is incompatible with rest of environment. Should use intel-mkl instead, which requires modifying compiler and linker flags. https://www.intel.com/content/www/us/en/developer/tools/oneapi/onemkl-link-line-advisor.html - intel-mkl provides FFTW compatibility, but only a subset of the functions. The FFTW plan we use is not supported.
CDO is included in the 2023r1 and 2023r1-gcc11 module sets.
module load 2023rc1-gcc11
module load openmpi/4.1.4
module load cdo/2.1.0
See here for how to use cdo for merging the output.
This is for installing cdo 2.1.0 by compiling from source (not needed anymore since the 2023 module sets have it).
module load 2022r2
module load openmpi/4.1.1
module load netcdf-c/4.8.1
module load hdf5/1.10.7
export CC=mpicc
export CXX=mpicxx
cd /scratch/$USER
wget https://code.mpimet.mpg.de/attachments/download/27481/cdo-2.1.0.tar.gz
tar -xzf cdo-2.1.0.tar.gz
cd cdo-2.1.0
./configure --with-netcdf=`nc-config --prefix` --with-hdf5=/apps/arch/2022r2/software/linux-rhel8-skylake_avx512/gcc-8.5.0/hdf5-1.10.7-wscpmjfq75bppp3geu4xtecw3buxhnke/
make -j 4
the program is now in src/cdo. copy it to your home directory, perhaps ~/bin/cdo so it doesn't get deleted from the scratch.
mkdir ~/bin
cp src/cdo ~/bin/
#add to .bashrc or similar:
export PATH=$PATH:/home/$USER/bin
now cdo is usable anywhere.
To use later in a new terminal, you may have to load the modules again:
module load 2022r2
module load openmpi/4.1.1
module load netcdf-c/4.8.1
module load hdf5/1.10.7
(2022) compiles but FFTW does not work, our fftw-plan is not supported.
module load intel-mkl
export SYST=gnu-fast-mkl
cmake .. -DUSE_FFTW=True
changes in CMakeList.txt:
elseif("$ENV{SYST}" STREQUAL "gnu-fast-mkl")
set(USE_MKL True)
set(FFTW_INCLUDE_DIR $ENV{MKLROOT}/include/fftw)
set(CMAKE_Fortran_COMPILER "mpif90")
set(CMAKE_Fortran_FLAGS "-cpp -W -Wall -fdefault-real-8 -fdefault-double-8 -march=native -ffree-line-length-none -std=gnu -Werror=implicit-interface -L$ENV{MKLROOT}/lib/intel64 -Wl,--no-as-needed -lmkl_gf_lp64 -lmkl_sequential -lmkl_core -lpthread -lm -ldl" CACHE STRING "")
set (CMAKE_Fortran_FLAGS_RELEASE "-funroll-all-loops -fno-f2c -Ofast -g -fbacktrace" CACHE STRING "")
set (CMAKE_Fortran_FLAGS_DEBUG "-finit-real=nan -fbounds-check -fbacktrace -fno-f2c -O0 -g -ffpe-trap=invalid,zero,overflow" CACHE STRING "")
---------------
if(USE_FFTW)
# Build the FFTW-based Poisson solver
if(USE_MKL)
#list(APPEND OPTIONAL_LIBS $ENV{MKLROOT}/lib/intel64/libmkl_gf_lp64.a $ENV{M
KLROOT}/lib/intel64/libmkl_sequential.a $ENV{MKLROOT}/lib/intel64/libmkl_core.a m dl)
else()
list(APPEND OPTIONAL_LIBS ${FFTW_LIB} ${FFTWF_LIB})
endif(USE_MKL)
set(opt_flags "${opt_flags} -DUSE_FFTW")
endif(USE_FFTW)