Levante/wiso

env.sh

@@ -27,6 +27,8 @@ fi
 
 if [[ $LOGINHOST =~ ^m[A-Za-z0-9]+\.hpc\.dkrz\.de$ ]]; then
    STRATEGY="mistral.dkrz.de"
+elif [[ $LOGINHOST =~ ^l[A-Za-z0-9]+\.atos\.local$ ]]; then
+   STRATEGY="levante.dkrz.de"
 elif [[ $LOGINHOST =~ ^ollie[0-9]$ ]] || [[ $LOGINHOST =~ ^prod-[0-9]{4}$ ]]; then
    STRATEGY="ollie"
 elif [[ $LOGINHOST =~ ^h[A-Za-z0-9]+\.hsn\.hlrn\.de$ ]]; then
@@ -60,5 +62,6 @@ if [ $BEING_EXECUTED = true ]; then
    echo $DIR/env/$STRATEGY
 else
    # file is being sourced
+   export FESOM_PLATFORM_STRATEGY=$STRATEGY
    source $DIR/env/$STRATEGY/shell
 fi

env/levante.dkrz.de/shell

@@ -0,0 +1,13 @@
+# make the contents as shell agnostic as possible so we can include them with bash, zsh and others
+export LC_ALL=en_US.UTF-8
+export CPU_MODEL=AMD_EPYC_ZEN3
+
+module load intel-oneapi-compilers/2022.0.1-gcc-11.2.0
+module load intel-oneapi-mkl/2022.0.1-gcc-11.2.0
+module load openmpi/4.1.2-intel-2021.5.0
+export FC=mpif90 CC=mpicc CXX=mpicxx ;
+export LD_LIBRARY_PATH=/sw/spack-levante/intel-oneapi-mkl-2022.0.1-ttdktf/mkl/2022.0.1/lib/intel64:$LD_LIBRARY_PATH
+
+module load netcdf-c/4.8.1-openmpi-4.1.2-intel-2021.5.0
+module load netcdf-fortran/4.5.3-openmpi-4.1.2-intel-2021.5.0
+module load git # to be able to determine the fesom git SHA when compiling

lib/parms/CMakeLists.txt

@@ -18,4 +18,7 @@ target_include_directories(${PROJECT_NAME}
 target_link_libraries(${PROJECT_NAME} INTERFACE ${BLAS_C_LIBRARIES})
 if(${CMAKE_C_COMPILER_ID} STREQUAL "Intel")
 	target_compile_options(${PROJECT_NAME} PRIVATE -no-prec-div -no-prec-sqrt -fast-transcendentals -fp-model precise)
+   if(${FESOM_PLATFORM_STRATEGY} STREQUAL  levante.dkrz.de )
+      target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -mtune=core-avx2)
+   endif()
 endif()

src/CMakeLists.txt

@@ -3,6 +3,12 @@ set(CMAKE_OSX_DEPLOYMENT_TARGET "10.9")
 
 project(fesom C Fortran)
 
+if(DEFINED ENV{FESOM_PLATFORM_STRATEGY})
+   set(FESOM_PLATFORM_STRATEGY $ENV{FESOM_PLATFORM_STRATEGY} CACHE STRING "switch to platform specific compile settings, this is usually determined via the env.sh script")
+else()
+   set(FESOM_PLATFORM_STRATEGY "notset" CACHE STRING "switch to platform specific compile settings, this is usually determined via the env.sh script")
+endif()
+
 option(DISABLE_MULTITHREADING "disable asynchronous operations" OFF)
 
 # get our source files
@@ -68,7 +74,12 @@ if(${VERBOSE})
 endif()
 # CMAKE_Fortran_COMPILER_ID will also work if a wrapper is being used (e.g. mpif90 wraps ifort -> compiler id is Intel)
 if(${CMAKE_Fortran_COMPILER_ID} STREQUAL  Intel )
-	target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -xHost -ip -init=zero -no-wrap-margin)
+	target_compile_options(${PROJECT_NAME} PRIVATE -r8 -i4 -fp-model precise -no-prec-div -no-prec-sqrt -fimf-use-svml -ip -init=zero -no-wrap-margin)
+   if(${FESOM_PLATFORM_STRATEGY} STREQUAL  levante.dkrz.de )
+      target_compile_options(${PROJECT_NAME} PRIVATE -march=core-avx2 -mtune=core-avx2)
+   else()
+      target_compile_options(${PROJECT_NAME} PRIVATE -xHost)
+   endif()
 elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL  GNU )
 	target_compile_options(${PROJECT_NAME} PRIVATE -O3 -finit-local-zero  -finline-functions -march=native -fimplicit-none  -fdefault-real-8 -ffree-line-length-none)
 	if(CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 10 )

work/job_ini_levante

@@ -0,0 +1,38 @@
+#!/bin/bash
+#SBATCH --job-name=ref
+#SBATCH -p compute
+#SBATCH --ntasks-per-node=1
+#SBATCH --ntasks=1
+#SBATCH --time=01:00:00
+#SBATCH -o slurm-out.out
+#SBATCH -e slurm-err.out
+#SBATCH -A ab0995
+
+source /sw/etc/profile.levante
+source ../env/levante.dkrz.de/shell
+
+ulimit -s 102400
+
+echo Submitted job: $jobid
+squeue -u $USER
+
+# determine JOBID
+JOBID=`echo $SLURM_JOB_ID |cut -d"." -f1`
+
+ln -s ../bin/fesom_ini.x .           # cp -n ../bin/fesom.x
+cp -n ../config/namelist.config  .
+cp -n ../config/namelist.forcing .
+cp -n ../config/namelist.oce     .
+cp -n ../config/namelist.ice     .
+cp -n ../config/namelist.icepack .
+
+date
+srun -l fesom_ini.x > "fesom_ini.out"
+date
+
+# qstat -f $PBS_JOBID
+#export EXITSTATUS=$?
+#if [ ${EXITSTATUS} -eq 0 ] || [ ${EXITSTATUS} -eq 127 ] ; then
+#sbatch job_mistral
+#fi
+

work/job_levante

@@ -0,0 +1,38 @@
+#!/bin/bash
+#SBATCH --job-name=ref
+#SBATCH -p compute
+#SBATCH --ntasks-per-node=128
+#SBATCH --ntasks=512
+#SBATCH --time=08:00:00
+#SBATCH -o slurm-out.out
+#SBATCH -e slurm-err.out
+#SBATCH -A ab0995
+
+source /sw/etc/profile.levante
+source ../env/levante.dkrz.de/shell
+
+ulimit -s 102400
+
+echo Submitted job: $jobid
+squeue -u $USER
+
+# determine JOBID
+JOBID=`echo $SLURM_JOB_ID |cut -d"." -f1`
+
+ln -s ../bin/fesom.x .           # cp -n ../bin/fesom.x
+cp -n ../config/namelist.config  .
+cp -n ../config/namelist.forcing .
+cp -n ../config/namelist.oce     .
+cp -n ../config/namelist.ice     .
+cp -n ../config/namelist.icepack .
+
+date
+srun -l fesom.x > "fesom2.0.out"
+date
+
+# qstat -f $PBS_JOBID
+#export EXITSTATUS=$?
+#if [ ${EXITSTATUS} -eq 0 ] || [ ${EXITSTATUS} -eq 127 ] ; then
+#sbatch job_mistral
+#fi
+