Merge pull request #59 from ecmwf-ifs/nams_lumi_hip

CLOUDSC HIP (SCC, SCC-HOIST, SCC-K-CACHING)

CMakeLists.txt

@@ -56,6 +56,20 @@ if( HAVE_CUDA )
     enable_language( CUDA )
 endif()
 
+ecbuild_add_option( FEATURE HIP
+	DESCRIPTION "HIP" DEFAULT OFF
+	REQUIRED_PACKAGES "hip" )
+if ( HAVE_HIP )
+  if(NOT DEFINED ROCM_PATH)
+    if(DEFINED ENV{ROCM_PATH})
+      set(ROCM_PATH $ENV{ROCM_PATH} CACHE PATH "Path to which ROCM has been installed")
+    else()
+      set(ROCM_PATH "/opt/rocm" CACHE PATH "Path to which ROCM has been installed")
+    endif()
+  endif()
+  find_package(hip REQUIRED)
+endif()
+
 ### OpenMP
 ecbuild_add_option( FEATURE OMP
     DESCRIPTION "OpenMP" DEFAULT ON

arch/eurohpc/lumi/cray-gpu/15.0.1/env.sh

@@ -0,0 +1,51 @@
+# (C) Copyright 1988- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+# Source me to get the correct configure/build/run environment
+
+# Store tracing and disable (module is *way* too verbose)
+{ tracing_=${-//[^x]/}; set +x; } 2>/dev/null
+
+module_load() {
+  echo "+ module load $1"
+  module load $1
+}
+module_unload() {
+  echo "+ module unload $1"
+  module unload $1
+}
+
+# Unload to be certain
+module reset
+
+# Load modules
+module_load PrgEnv-cray/8.3.3
+module_load LUMI/23.03
+# module_load partition/G
+module_load rocm/5.2.3
+module_load cce/15.0.1
+module_load cray-libsci/22.08.1.1
+module_load cray-mpich/8.1.18
+module_load craype/2.7.20
+module_load craype-accel-amd-gfx90a
+module_load buildtools/23.03
+module_load cray-hdf5/1.12.1.5
+module_load cray-python/3.9.12.1
+module_load Boost/1.81.0-cpeCray-23.03
+module_load partition/G
+
+module list
+
+set -x
+
+export CC=cc CXX=CC FC=ftn
+
+# Restore tracing to stored setting
+{ if [[ -n "$tracing_" ]]; then set -x; else set +x; fi } 2>/dev/null
+
+export ECBUILD_TOOLCHAIN="./toolchain.cmake"

arch/eurohpc/lumi/cray-gpu/15.0.1/toolchain.cmake

@@ -0,0 +1,58 @@
+# (C) Copyright 1988- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+####################################################################
+# COMPILER
+####################################################################
+
+set( ECBUILD_FIND_MPI OFF )
+set( ENABLE_USE_STMT_FUNC ON CACHE STRING "" )
+
+####################################################################
+# OpenMP FLAGS
+####################################################################
+
+set( ENABLE_OMP ON CACHE STRING "" )
+set( OpenMP_C_FLAGS   "-fopenmp" CACHE STRING "" )
+set( OpenMP_CXX_FLAGS "-fopenmp" CACHE STRING "" )
+set( OpenMP_Fortran_FLAGS   "-fopenmp -hnoacc -hlist=aimd" CACHE STRING "" )
+
+set( OpenMP_C_LIB_NAMES       "craymp" )
+set( OpenMP_CXX_LIB_NAMES     "craymp" )
+set( OpenMP_Fortran_LIB_NAMES "craymp" )
+set( OpenMP_craymp_LIBRARY    "/opt/cray/pe/cce/15.0.1/cce/x86_64/lib/libcraymp.so" )
+
+####################################################################
+# OpenACC FLAGS
+####################################################################
+
+set( ENABLE_ACC ON CACHE STRING "" )
+set( OpenACC_C_FLAGS "-hacc" )
+set( OpenACC_CXX_FLAGS "-hacc" )
+set( OpenACC_Fortran_FLAGS "-hacc -h acc_model=deep_copy" )
+
+####################################################################
+# OpenACC FLAGS
+####################################################################
+
+set(CMAKE_HIP_FLAGS "${CMAKE_HIP_FLAGS} -03 -ffast-math")
+if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
+  set(CMAKE_HIP_ARCHITECTURES gfx90a)
+endif()
+
+####################################################################
+# Compiler FLAGS
+####################################################################
+
+# General Flags (add to default)
+set(ECBUILD_Fortran_FLAGS "-hcontiguous")
+set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -hbyteswapio")
+set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -Wl, --as-needed")
+
+set(ECBUILD_Fortran_FLAGS_BIT "-O3 -G2 -haggress -DNDEBUG")
+# set(ECBUILD_Fortran_FLAGS_BIT "-O3 -hfp1 -hscalar3 -hvector3 -G2 -haggress -DNDEBUG")

bundle.yml

@@ -109,6 +109,11 @@ options :
             ENABLE_CLOUDSC_GPU_SCC_CUF=ON
             ENABLE_CLOUDSC_GPU_SCC_CUF_K_CACHING=ON
             BUILD_field_api=ON
+    
+    - with-hip :
+        help: Enable GPU kernel variant based on HIP
+        cmake: >
+            ENABLE_HIP=ON 
 
     - with-mpi :
         help  : Enable MPI-parallel kernel

src/CMakeLists.txt

@@ -14,5 +14,6 @@ add_subdirectory(cloudsc_pyiface)
 add_subdirectory(cloudsc_python)
 add_subdirectory(cloudsc_c)
 add_subdirectory(cloudsc_cuda)
+add_subdirectory(cloudsc_hip)
 add_subdirectory(cloudsc_gpu)
 add_subdirectory(cloudsc_loki)

src/cloudsc_hip/CMakeLists.txt

@@ -0,0 +1,171 @@
+# (C) Copyright 1988- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+# Define this dwarf variant as an ECBuild feature
+ecbuild_add_option( FEATURE CLOUDSC_HIP
+	DESCRIPTION "Build the HIP version CLOUDSC using Serialbox" DEFAULT ON
+	CONDITION Serialbox_FOUND AND HAVE_HIP
+)
+
+if( HAVE_CLOUDSC_HIP )
+
+    set(CMAKE_C_COMPILER "${ROCM_PATH}/bin/hipcc")
+    set(CMAKE_CXX_COMPILER "${ROCM_PATH}/bin/hipcc")
+
+    ###### SCC-HIP ####
+    ecbuild_add_library(
+        TARGET dwarf-cloudsc-hip-lib
+    	INSTALL_HEADERS LISTED
+        SOURCES
+    	    cloudsc/yoecldp_c.h
+    	    cloudsc/load_state.h
+            cloudsc/load_state.cpp
+            cloudsc/cloudsc_c.h
+            cloudsc/cloudsc_c.cpp
+            cloudsc/cloudsc_driver.h
+            cloudsc/cloudsc_driver.cpp
+            cloudsc/cloudsc_validate.h
+            cloudsc/cloudsc_validate.cpp
+            cloudsc/mycpu.h
+            cloudsc/mycpu.cpp
+        PUBLIC_INCLUDES
+            $<INSTALL_INTERFACE:include>
+            $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cloudsc>
+        PUBLIC_LIBS
+    	    hip::device
+            Serialbox::Serialbox_C
+            $<${HAVE_OMP}:OpenMP::OpenMP_C>
+    )
+
+    target_include_directories(dwarf-cloudsc-hip-lib PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cloudsc>)
+    target_link_libraries(dwarf-cloudsc-hip-lib PUBLIC hip::device Serialbox::Serialbox_C $<${HAVE_OMP}:OpenMP::OpenMP_C>)
+
+    if (NOT DEFINED CMAKE_HIP_ARCHITECTURES)
+      message(WARNING "No HIP architecture is set! ('CMAKE_HIP_ARCHITECTURES' is not defined)")
+    else()
+      target_compile_options(dwarf-cloudsc-hip-lib PRIVATE --offload-arch=${CMAKE_HIP_ARCHITECTURES})
+    endif()
+
+    ecbuild_add_executable(
+        TARGET  dwarf-cloudsc-hip
+        SOURCES dwarf_cloudsc.cpp
+        LIBS dwarf-cloudsc-hip-lib
+    )
+
+    ecbuild_add_test(
+        TARGET dwarf-cloudsc-hip-serial
+        COMMAND bin/dwarf-cloudsc-hip
+        ARGS 1 1000 64
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../..
+        OMP 1
+    )
+    ## 
+
+    ###### SCC-HOIST-HIP ####
+    ecbuild_add_library(
+        TARGET dwarf-cloudsc-hip-hoist-lib
+    	INSTALL_HEADERS LISTED
+        SOURCES
+    	    cloudsc/yoecldp_c.h
+    	    cloudsc/load_state.h
+            cloudsc/load_state.cpp
+            cloudsc/cloudsc_c_hoist.h
+            cloudsc/cloudsc_c_hoist.cpp
+            cloudsc/cloudsc_driver_hoist.h
+            cloudsc/cloudsc_driver_hoist.cpp
+            cloudsc/cloudsc_validate.h
+            cloudsc/cloudsc_validate.cpp
+            cloudsc/mycpu.h
+            cloudsc/mycpu.cpp
+        PUBLIC_INCLUDES
+            $<INSTALL_INTERFACE:include>
+            $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cloudsc>
+        PUBLIC_LIBS
+    	    hip::device
+            Serialbox::Serialbox_C
+            $<${HAVE_OMP}:OpenMP::OpenMP_C>
+    )
+
+    target_include_directories(dwarf-cloudsc-hip-hoist-lib PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cloudsc>)
+    target_link_libraries(dwarf-cloudsc-hip-hoist-lib PUBLIC hip::device Serialbox::Serialbox_C $<${HAVE_OMP}:OpenMP::OpenMP_C>)
+
+    if (NOT DEFINED CMAKE_HIP_ARCHITECTURES)
+      message(WARNING "No HIP architecture is set! ('CMAKE_HIP_ARCHITECTURES' is not defined)")
+    else()
+      target_compile_options(dwarf-cloudsc-hip-hoist-lib PRIVATE --offload-arch=${CMAKE_HIP_ARCHITECTURES})
+    endif()
+
+    ecbuild_add_executable(
+        TARGET  dwarf-cloudsc-hip-hoist
+        SOURCES dwarf_cloudsc.cpp
+        LIBS dwarf-cloudsc-hip-hoist-lib
+    )
+
+    ecbuild_add_test(
+        TARGET dwarf-cloudsc-hip-hoist-serial
+        COMMAND bin/dwarf-cloudsc-hip-hoist
+        ARGS 1 1000 64
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../..
+        OMP 1
+    )
+    ## 
+
+    ###### SCC-K-CACHING-HIP ####
+    ecbuild_add_library(
+        TARGET dwarf-cloudsc-hip-k-caching-lib
+    	INSTALL_HEADERS LISTED
+        SOURCES
+    	    cloudsc/yoecldp_c.h
+    	    cloudsc/load_state.h
+            cloudsc/load_state.cpp
+            cloudsc/cloudsc_c_k_caching.h
+            cloudsc/cloudsc_c_k_caching.cpp
+            cloudsc/cloudsc_driver.h
+            cloudsc/cloudsc_driver.cpp
+            cloudsc/cloudsc_validate.h
+            cloudsc/cloudsc_validate.cpp
+            cloudsc/mycpu.h
+            cloudsc/mycpu.cpp
+        PUBLIC_INCLUDES
+            $<INSTALL_INTERFACE:include>
+            $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cloudsc>
+        PUBLIC_LIBS
+    	    hip::device
+            Serialbox::Serialbox_C
+            $<${HAVE_OMP}:OpenMP::OpenMP_C>
+    )
+
+    target_include_directories(dwarf-cloudsc-hip-k-caching-lib PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cloudsc>)
+    target_link_libraries(dwarf-cloudsc-hip-k-caching-lib PUBLIC hip::device Serialbox::Serialbox_C $<${HAVE_OMP}:OpenMP::OpenMP_C>)
+
+    if (NOT DEFINED CMAKE_HIP_ARCHITECTURES)
+      message(WARNING "No HIP architecture is set! ('CMAKE_HIP_ARCHITECTURES' is not defined)")
+    else()
+      target_compile_options(dwarf-cloudsc-hip-k-caching-lib PRIVATE --offload-arch=${CMAKE_HIP_ARCHITECTURES})
+    endif()
+
+    ecbuild_add_executable(
+        TARGET  dwarf-cloudsc-hip-k-caching
+        SOURCES dwarf_cloudsc.cpp
+        LIBS dwarf-cloudsc-hip-k-caching-lib
+    )
+
+    ecbuild_add_test(
+        TARGET dwarf-cloudsc-hip-k-caching-serial
+        COMMAND bin/dwarf-cloudsc-hip-k-caching
+        ARGS 1 1000 64
+        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../../..
+        OMP 1
+    )
+    ## 
+
+    # Create symlink for the input data
+    execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink
+        ${CMAKE_CURRENT_SOURCE_DIR}/../../data ${CMAKE_CURRENT_BINARY_DIR}/../../../data )
+
+endif()