CMakeLists.txt

cmake_minimum_required(VERSION 3.18)
project(MPLB C CXX)
option(VERBOSE "Turn on verbose warning messages" OFF)
option(OPTIMISE "Turn on optimised mode" OFF)
#option(TEST "Turn on tests for Apps" OFF)
if (NOT VERBOSE)
    message("We show concise compiling information by defautl! Use -DVERBOSE=ON to switch on.")
endif()
if (NOT OPTIMISE)
    message("We use the development mode by defautl! Use -DOPTIMISE=ON to use the optimised mode.")
endif()
set(CMAKE_VERBOSE_MAKEFILE ${VERBOSE})
set(LibDir ${CMAKE_SOURCE_DIR}/Src)
# Use the Release mode by default
if ( NOT CMAKE_BUILD_TYPE )
    message("We choose the Release mode! Please uses -DCMAKE_BUILD_TYPE to change.")
    set(CMAKE_BUILD_TYPE Release)
endif()
# Configure Compilers
# C
set(CMAKE_C_STANDARD 99)
#TODO:Shall we keep the "-g" in the release mode? It increases file size.
if (${CMAKE_C_COMPILER_ID} STREQUAL GNU)
    set(CMAKE_C_FLAGS "-fPIC -Wall")
    set(CMAKE_C_FLAGS_RELEASE "-O3")
    set(CMAKE_C_FLAGS_DEBUG "-O0 -g")
elseif (${CMAKE_C_COMPILER_ID} STREQUAL Clang)
    set(CMAKE_C_FLAGS "-fPIC -DUNIX -Wall")
    set(CMAKE_C_FLAGS_RELEASE "-O3")
    set(CMAKE_C_FLAGS_DEBUG "-g -O0")
elseif (${CMAKE_C_COMPILER_ID} STREQUAL MSVC)
    set(CMAKE_C_FLAGS "-W1")
    set(CMAKE_C_FLAGS_RELEASE "/O2 /Ot ")
    set(CMAKE_C_FLAGS_DEBUG "/Z7 /DEBUG")
elseif (${CMAKE_C_COMPILER_ID} STREQUAL Intel)
    set(CMAKE_C_FLAGS "-xHost")
    set(CMAKE_C_FLAGS_RELEASE "-O3")
    set(CMAKE_C_FLAGS_DEBUG "-O0 -g")
else ()
    message(WARNING "There is no default flag for the chosen C compiler! Please use -DCFLAG=XXX to supply!")
endif ()

# C++
set(CMAKE_CXX_STANDARD 11)
if (${CMAKE_CXX_COMPILER_ID} STREQUAL GNU)
    set(CMAKE_CXX_FLAGS "-fPIC -Wall")
    set(CMAKE_CXX_FLAGS_RELEASE "-O3")
    set(CMAKE_CXX_FLAGS_DEBUG "-g -O0")
elseif (${CMAKE_CXX_COMPILER_ID} STREQUAL Clang)
    set(CMAKE_CXX_FLAGS "-fPIC -DUNIX -Wall")
    set(CMAKE_CXX_FLAGS_RELEASE "-O3")
    set(CMAKE_CXX_FLAGS_DEBUG "-g -O0")
elseif (${CMAKE_CXX_COMPILER_ID} STREQUAL MSVC)
    set(CMAKE_CXX_FLAGS "-W1")
    set(CMAKE_CXX_FLAGS_RELEASE "/O2 /Ot ")
    set(CMAKE_CXX_FLAGS_DEBUG "/Z7 /DEBUG")
elseif (${CMAKE_CXX_COMPILER_ID} STREQUAL Intel)
    set(CMAKE_CXX_FLAGS "-xHost")
    set(CMAKE_CXX_FLAGS_RELEASE "-O3")
    set(CMAKE_CXX_FLAGS_DEBUG " -O0 -g -DOPS_DEBUG ")
else ()
    message(WARNING "There is no default flag for the chosen C++ compiler! Please use -DCXXFLAG=XXX to supply!")
endif ()

if (CFLAG)
    SET(CMAKE_C_FLAGS  "${CMAKE_C_FLAGS} ${CFLAG}")
else ()
    message("Extra C compiler options by using CFLAG, e.g., -DCFLAG=\"-ffloat-store\" for GNU!")
endif ()

if (CXXFLAG)
    SET(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} ${CXXFLAG}")
else ()
    message("Extra C++ compiler options by using CXXFLAG, e.g., -DCXXFLAG=\"-ffloat-store\" for GNU!")
endif ()
# Try to find the required dependency
set(HDF5_PREFER_PARALLEL true)
find_package(MPI QUIET)
find_package(HDF5 QUIET COMPONENTS C HL)
# Configure the "include" dir for compiling

if (NOT HDF5_FOUND)
    message (WARNING "We cannot find the parallel HDF5 library. The HDF5 IO routines won't work! Please use -DHDF5_ROOT to specify the path!")
endif ()
if (NOT MPI_FOUND)
    message (WARNING "We cannot find a MPI environment! The MPI codes won't compile!")
endif ()
if (OPTIMISE)
    find_package(CUDAToolkit QUIET)
    find_package(OpenACC QUIET)
    find_package(OpenCL QUIET)
    find_package(OpenMP QUIET)
    find_package(Python2 QUIET)
    if (NOT Python2_FOUND)
        message (FATAL_ERROR "We cannot find Python2 and the Python translator needs Python2! Please use -DPython2_EXECUTABLE to specify the path.")
    else()
        FIND_PATH (opsc NAMES "ops.py" PATHS ${OPS_ROOT} PATH_SUFFIXES bin/ops_translator/c)
        FIND_PATH (opsfortran NAMES "ops_fortran.py" PATHS ${OPS_ROOT} PATH_SUFFIXES bin/ops_translator/fortran)
        if (NOT opsc)
            message (FATAL_ERROR "We cannot find the python C/C++ translator!")
        endif()
        if (NOT opsfortran)
            message (FATAL_ERROR "We cannot find the python fortran translator!")
        endif()
        set(OPS_C_TRANSLATOR  "${opsc}/ops.py")
        set(OPS_F_TRANSLATOR  "${opsfortran}/ops_fortran.py")
    endif () # python2

    if (CUDAToolkit_FOUND)
        set(CMAKE_CUDA_COMPILER ${CUDAToolkit_NVCC_EXECUTABLE})
        if (GPU_ARCH)
            set(CMAKE_CUDA_ARCHITECTURES 70  CACHE STRING "CUDA architectures")
        else()
        message(WARNING "Please the GPU architecture using -DGPU_ARCH=XXX!")
    endif()
        set(CMAKE_CUDA_FLAGS_DEBUG " -G ${CMAKE_CUDA_FLAGS_DEBUG}")
        enable_language(CUDA)
    else()
        message (WARNING "We cannot find a CUDA kit! The CUDA codes won't compile!")
    endif ()

    if (OpenCL_FOUND)
        include_directories(${OpenCL_INCLUDE_DIRS})
    else()
        message (WARNING "We cannot find a OpenCL kit! The OpenCL codes won't compile!")
    endif ()
endif () # optimise
if (OPS_ROOT)
    set(OPS_DIR ${OPS_ROOT})
    find_package(OPS PATHS ${OPS_DIR}/lib/cmake)
else()
    find_package(OPS)
endif()
if (NOT OPS_FOUND)
    message(FATAL_ERROR "We cannot find the OPS library! If it is installed, please specify its directory by using -DOPS_ROOT=XXX.")
endif()

set(HDF5_SEQ ${HDF5_FOUND})
if (OPTIMISE)
    set(CUDA ${CUDAToolkit_FOUND})
    set(OPENCL ${OpenCL_FOUND})
endif()

if (MPI_FOUND)
    set(MPI TRUE)
    set(HDF5_MPI ${HDF5_FOUND})
    if (OPTIMISE)
        set(CUDA_MPI ${CUDAToolkit_FOUND})
        set(OPENCL_MPI ${OpenCL_FOUND})
    endif()
endif()
#prepare ctest
if (TEST)
    enable_testing()
    include(ProcessorCount)
    ProcessorCount(CPU_NUMBER)
    if(CPU_NUMBER GREATER 12)
        set(CPU_NUMBER 12)
    endif()
    if (NOT GPU_NUMBER)
        message(WARNING "GPU tests are disabled! Please use -DGPU_NUMBER to specify!")
    endif()
endif()

# clean the tmp directory for the code generation
macro(CreateTempDir)
    if(EXISTS ${CMAKE_CURRENT_BINARY_DIR}/tmp)
        file(REMOVE_RECURSE "${CMAKE_CURRENT_BINARY_DIR}/tmp")
    endif(EXISTS ${CMAKE_CURRENT_BINARY_DIR}/tmp )
    file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/tmp" )
endmacro()

macro(SeqDevTarget SpaceDim DebugLevel)
    add_executable(${AppName}SeqDev ${LibSrcPath} ${AppSrc})
    target_include_directories(${AppName}SeqDev PRIVATE ${LibDir} ${CMAKE_CURRENT_SOURCE_DIR})
    target_link_libraries(${AppName}SeqDev OPS::ops_hdf5_seq OPS::ops_seq hdf5::hdf5 hdf5::hdf5_hl MPI::MPI_CXX)
    target_compile_definitions(${AppName}SeqDev PRIVATE -DOPS_${SpaceDim}D -DCPU -DLEVEL=DebugLevel=${DebugLevel})
endmacro(SeqDevTarget DebugLevel)

macro(MpiDevTarget SpaceDim DebugLevel)
    if (MPI)
        add_executable(${AppName}MpiDev ${LibSrcPath} ${AppSrc})
        target_include_directories(${AppName}MpiDev PRIVATE ${LibDir} ${CMAKE_CURRENT_SOURCE_DIR})
        target_link_libraries(${AppName}MpiDev OPS::ops_hdf5_mpi OPS::ops_mpi hdf5::hdf5 hdf5::hdf5_hl MPI::MPI_CXX)
        target_compile_definitions(${AppName}MpiDev PRIVATE -DOPS_${SpaceDim}D -DOPS_MPI -DCPU -DLEVEL=DebugLevel=${DebugLevel} )
    endif()
endmacro(MpiDevTarget DebugLevel)

# The files needed to be translated by ops.py from the library side
set(LibSrcGenList boundary_wrapper.cpp flowfield_wrapper.cpp model_wrapper.cpp scheme_wrapper.cpp model.cpp)
set(LibKernelGenList boundary_kernel.inc flowfield_kernel.inc model_kernel.inc scheme_kernel.inc)

function (WriteJsonConfig Dir AppName LibSrc AppSrcGenList AppKernelGenList HeadList SpaceDim)
    set(SourceKey "\"source\":[" )
    set(LibGenList "${LibSrcGenList}")
    foreach(LibGen IN LISTS LibGenList)
        if (NOT (LibGen IN_LIST LibSrc))
            list(REMOVE_ITEM LibGenList ${LibGen})
        endif()
    endforeach()

    foreach(Source IN LISTS LibGenList AppSrcGenList)
        string(APPEND SourceKey "\"${Source}\",")
    endforeach()
    string(LENGTH ${SourceKey} Len)
    math(EXPR Len "${Len}-1")
    string(SUBSTRING ${SourceKey} 0 ${Len} SourceKey)
    string(APPEND SourceKey "],")

    set(KernelKey "\"kernel\":[" )
    foreach(Kernel IN LISTS LibKernelGenList AppKernelGenList)
        string(APPEND KernelKey "\"${Kernel}\",")
    endforeach()
    string(LENGTH ${KernelKey} Len)
    math(EXPR Len "${Len}-1")
    string(SUBSTRING ${KernelKey} 0 ${Len} KernelKey)
    string(APPEND KernelKey "],")
    list(LENGTH HeadList HeadNum)
    if (${HeadNum} LESS 1)
       set(HeadKey "")
    else()
        set(HeadKey "\"head\":[" )
        foreach(Head IN LISTS HeadList)
            string(APPEND HeadKey "\"${Head}\",")
        endforeach()
        string(LENGTH ${HeadKey} Len)
        math(EXPR Len "${Len}-1")
        string(SUBSTRING ${HeadKey} 0 ${Len} HeadKey)
        string(APPEND HeadKey "],")
    endif()

    set(SpaceDimKey "\"spacedim\": ${SpaceDim},")
    set(CaseKey "\"case\": \"${AppName}\",")
    set(BackendKey "\"backend\":[\"mpi_lazy\",\"mpi_cuda\"]")
    set(TypeDefKey "\"typedef\": { \"Real\": \"double\"},")

    file(WRITE "${Dir}/OPSPYConfig.json" "{ ${CaseKey} ${SourceKey} ${KernelKey} ${TypeDefKey} ${HeadKey} ${SpaceDimKey} ${BackendKey} }")
endfunction(WriteJsonConfig AppName AppSrcGenList AppKernelGenList AppHeadGenList SpaceDim)

macro (TranslateSourceCodes LibDir LibSrcGenList AppSrcGenList Destination)
    file(GLOB LibFiles  ${LibDir}/*.cpp ${LibDir}/*.h ${LibDir}/*.inc ${LibDir}/*.hpp)
    file(COPY ${LibFiles} DESTINATION ${Destination})
    file(GLOB AppFiles  *.cpp *.h *.inc *.hpp)
    file(COPY ${AppFiles} DESTINATION ${Destination})
    execute_process (
        COMMAND ${OPS_C_TRANSLATOR}
        WORKING_DIRECTORY ${Destination}
    )
    set(TranslatedSrc "")
    set(LibSrcCopy ${LibSrc})
    foreach(Src IN LISTS LibSrcCopy)
        if (Src IN_LIST LibSrcGenList)
            list(REMOVE_ITEM LibSrc ${Src})
            get_filename_component(SrcName "${Src}" NAME_WE)
            list(APPEND TranslatedSrc ${Destination}/${SrcName}_ops.cpp)
        endif()
    endforeach()

    set(TmpSrcPath "")
    foreach(Src IN LISTS LibSrc LibTranslated)
        list(APPEND TmpSrcPath ${Destination}/${Src})
    endforeach(Src IN LISTS LibSrc LibTranslated)
    list(REMOVE_ITEM AppSrc ${AppSrcGenList})
    set(TmpAppSrcPath "")
    foreach(Src IN LISTS AppSrc)
        list(APPEND TmpAppSrcPath ${Destination}/${Src})
    endforeach(Src IN LISTS AppSrc)
    foreach(Src IN LISTS AppSrcGenList)
        get_filename_component(SrcName ${Src} NAME_WLE)
        list(APPEND TranslatedSrc ${Destination}/${SrcName}_ops.cpp)
    endforeach(Src IN LISTS AppSrc)
endmacro(TranslateSourceCodes  LibDir LibSrcGenList AppSrcGenList Destination)

macro(SeqTarget SpaceDim)
    add_executable(${AppName}Seq ${TMP_SOURCE_DIR}/MPI_OpenMP/${AppName}_cpu_kernels.cpp  ${TmpSrcPath} ${TmpAppSrcPath} ${TranslatedSrc})
    target_include_directories(${AppName}Seq PRIVATE ${TMP_SOURCE_DIR})
    target_link_libraries(${AppName}Seq PRIVATE OPS::ops_hdf5_seq OPS::ops_seq hdf5::hdf5 hdf5::hdf5_hl MPI::MPI_CXX)
    target_compile_definitions(${AppName}Seq PRIVATE -DOPS_${SpaceDim}D -DLEVEL=DebugLevel=0)
endmacro(SeqTarget)

macro(MpiTarget SpaceDim)
    if (MPI)
        add_executable(${AppName}Mpi ${TMP_SOURCE_DIR}/MPI_OpenMP/${AppName}_cpu_kernels.cpp  ${TmpSrcPath} ${TmpAppSrcPath} ${TranslatedSrc})
        target_include_directories(${AppName}Mpi PRIVATE ${TMP_SOURCE_DIR})
        target_link_libraries(${AppName}Mpi PRIVATE OPS::ops_hdf5_mpi OPS::ops_mpi hdf5::hdf5 hdf5::hdf5_hl MPI::MPI_CXX)
        target_compile_definitions(${AppName}Mpi PRIVATE -DOPS_${SpaceDim}D -DOPS_MPI -DLEVEL=DebugLevel=0 )
    endif()
endmacro(MpiTarget)

macro(CudaTarget SpaceDim)
if (CUDAToolkit_FOUND)
    add_executable(${AppName}Cuda ${TMP_SOURCE_DIR}/CUDA/${AppName}_kernels.cu  ${TmpSrcPath} ${TmpAppSrcPath} ${TranslatedSrc})
    set_property(TARGET ${AppName}Cuda PROPERTY CUDA_STANDARD 11)
    target_include_directories(${AppName}Cuda PRIVATE ${TMP_SOURCE_DIR})
    target_link_libraries(${AppName}Cuda PRIVATE OPS::ops_hdf5_seq OPS::ops_cuda CUDA::cudart_static hdf5::hdf5 hdf5::hdf5_hl MPI::MPI_CXX)
    target_compile_definitions(${AppName}Cuda PRIVATE -DOPS_${SpaceDim}D -DLEVEL=DebugLevel=0)
endif()
endmacro(CudaTarget)
add_subdirectory(Apps/3DCavity)
add_subdirectory(Apps/2DCavity)
add_subdirectory(Apps/3DLChannel)
add_subdirectory(Tests/FieldBlock)