From a9eef08e69dbab0dc8743eaa630bda3e7948c362 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 23 Oct 2023 21:57:29 -0400 Subject: [PATCH 1/6] initial import --- .jenkins/lsu/env-clang-12.sh | 2 + .jenkins/lsu/env-clang-13.sh | 2 + .jenkins/lsu/env-clang-14.sh | 2 + .jenkins/lsu/env-clang-15.sh | 2 + .jenkins/lsu/env-gcc-10.sh | 2 + .jenkins/lsu/env-gcc-12.sh | 2 + .jenkins/lsu/env-gcc-13.sh | 2 + CMakeLists.txt | 20 + cmake/FindOpenShmemPmi.cmake | 65 ++ cmake/HPX_SetupOpenSHMEM.cmake | 1002 +++++++++++++++++ cmake/templates/HPXConfig.cmake.in | 5 + cmake/templates/hpxrun.py.in | 11 + libs/core/CMakeLists.txt | 1 + libs/core/openshmem_base/CMakeLists.txt | 31 + .../openshmem_base/examples/CMakeLists.txt | 16 + .../include/hpx/openshmem_base/openshmem.hpp | 36 + .../openshmem_base/openshmem_environment.hpp | 126 +++ .../src/openshmem_environment.cpp | 378 +++++++ libs/core/openshmem_base/tests/CMakeLists.txt | 44 + .../tests/performance/CMakeLists.txt | 5 + .../tests/regressions/CMakeLists.txt | 5 + .../openshmem_base/tests/unit/CMakeLists.txt | 5 + libs/core/version/CMakeLists.txt | 7 + libs/core/version/src/version.cpp | 37 + libs/full/CMakeLists.txt | 1 + .../full/command_line_handling/CMakeLists.txt | 5 + .../src/command_line_handling.cpp | 15 + libs/full/parcelport_openshmem/CMakeLists.txt | 45 + .../examples/CMakeLists.txt | 18 + .../hpx/parcelport_openshmem/header.hpp | 150 +++ .../hpx/parcelport_openshmem/locality.hpp | 71 ++ .../hpx/parcelport_openshmem/receiver.hpp | 170 +++ .../receiver_connection.hpp | 269 +++++ .../hpx/parcelport_openshmem/sender.hpp | 164 +++ .../sender_connection.hpp | 325 ++++++ .../hpx/parcelport_openshmem/tag_provider.hpp | 61 + .../parcelport_openshmem/src/locality.cpp | 36 + .../src/parcelport_openshmem.cpp | 313 +++++ .../parcelport_openshmem/tests/CMakeLists.txt | 42 + .../tests/performance/CMakeLists.txt | 5 + .../tests/regressions/CMakeLists.txt | 5 + .../tests/unit/CMakeLists.txt | 5 + 42 files changed, 3508 insertions(+) create mode 100644 cmake/FindOpenShmemPmi.cmake create mode 100644 cmake/HPX_SetupOpenSHMEM.cmake create mode 100644 libs/core/openshmem_base/CMakeLists.txt create mode 100644 libs/core/openshmem_base/examples/CMakeLists.txt create mode 100644 libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp create mode 100644 libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp create mode 100644 libs/core/openshmem_base/src/openshmem_environment.cpp create mode 100644 libs/core/openshmem_base/tests/CMakeLists.txt create mode 100644 libs/core/openshmem_base/tests/performance/CMakeLists.txt create mode 100644 libs/core/openshmem_base/tests/regressions/CMakeLists.txt create mode 100644 libs/core/openshmem_base/tests/unit/CMakeLists.txt create mode 100644 libs/full/parcelport_openshmem/CMakeLists.txt create mode 100644 libs/full/parcelport_openshmem/examples/CMakeLists.txt create mode 100644 libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp create mode 100644 libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/locality.hpp create mode 100644 libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp create mode 100644 libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp create mode 100644 libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp create mode 100644 libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp create mode 100644 libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/tag_provider.hpp create mode 100644 libs/full/parcelport_openshmem/src/locality.cpp create mode 100644 libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp create mode 100644 libs/full/parcelport_openshmem/tests/CMakeLists.txt create mode 100644 libs/full/parcelport_openshmem/tests/performance/CMakeLists.txt create mode 100644 libs/full/parcelport_openshmem/tests/regressions/CMakeLists.txt create mode 100644 libs/full/parcelport_openshmem/tests/unit/CMakeLists.txt diff --git a/.jenkins/lsu/env-clang-12.sh b/.jenkins/lsu/env-clang-12.sh index 15e920431f54..acacb7d4783a 100644 --- a/.jenkins/lsu/env-clang-12.sh +++ b/.jenkins/lsu/env-clang-12.sh @@ -27,6 +27,8 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" diff --git a/.jenkins/lsu/env-clang-13.sh b/.jenkins/lsu/env-clang-13.sh index 4582dd0dba9f..3ccccb144588 100644 --- a/.jenkins/lsu/env-clang-13.sh +++ b/.jenkins/lsu/env-clang-13.sh @@ -27,6 +27,8 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index 5a52f3c2dca5..d41749efe60b 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -27,6 +27,8 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index ccb1e962fd0b..356925887f0d 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -27,6 +27,8 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" diff --git a/.jenkins/lsu/env-gcc-10.sh b/.jenkins/lsu/env-gcc-10.sh index 3953fae9420a..2a89f0e9b7a8 100644 --- a/.jenkins/lsu/env-gcc-10.sh +++ b/.jenkins/lsu/env-gcc-10.sh @@ -27,6 +27,8 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" diff --git a/.jenkins/lsu/env-gcc-12.sh b/.jenkins/lsu/env-gcc-12.sh index 1ae1a867f932..ee0661eb2ed5 100644 --- a/.jenkins/lsu/env-gcc-12.sh +++ b/.jenkins/lsu/env-gcc-12.sh @@ -26,6 +26,8 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" diff --git a/.jenkins/lsu/env-gcc-13.sh b/.jenkins/lsu/env-gcc-13.sh index 1454588eec09..8430b9ac01ba 100644 --- a/.jenkins/lsu/env-gcc-13.sh +++ b/.jenkins/lsu/env-gcc-13.sh @@ -26,6 +26,8 @@ configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" +configure_extra_options+=" -DHPX_WITH_PARCELPORT_OPENSHMEM=ON" +configure_extra_options+=" -DHPX_WITH_FETCH_OPENSHMEM=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c934e21328b..8c27847f7621 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1191,6 +1191,26 @@ if(HPX_WITH_NETWORKING) ) endif() + hpx_option( + HPX_WITH_PARCELPORT_OPENSHMEM BOOL "Enable the OPENSHMEM based parcelport." + OFF CATEGORY "Parcelport" + ) + hpx_option( + HPX_WITH_FETCH_OPENSHMEM + BOOL + "Download an OpenSHMEM if one is not available" OFF + CATEGORY "Build Targets" + ADVANCED + ) + if(HPX_WITH_PARCELPORT_OPENSHMEM) + hpx_add_config_define(HPX_HAVE_PARCELPORT_OPENSHMEM) + hpx_option( + HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT STRING + "Define which conduit to use for the OpenSHMEM parcelport" "sos" + STRINGS "sos;ucx" + ) + endif() + hpx_option( HPX_WITH_PARCELPORT_TCP BOOL "Enable the TCP based parcelport." ON CATEGORY "Parcelport" diff --git a/cmake/FindOpenShmemPmi.cmake b/cmake/FindOpenShmemPmi.cmake new file mode 100644 index 000000000000..5f6814a509db --- /dev/null +++ b/cmake/FindOpenShmemPmi.cmake @@ -0,0 +1,65 @@ +# Copyright (c) 2023 Christopher Taylor +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +find_package(PkgConfig QUIET) +# look for cray pmi... +pkg_check_modules(PC_PMI_CRAY QUIET cray-pmi) +# look for the rest if we couldn't find the cray package +if(NOT PC_PMI_CRAY_FOUND) + pkg_check_modules(PC_PMI QUIET pmi) +endif() + +find_path( + PMI_INCLUDE_DIR pmi2.h + HINTS ${PMI_ROOT} + ENV + PMI_ROOT + ${PMI_DIR} + ENV + PMI_DIR + ${PC_PMI_CRAY_INCLUDEDIR} + ${PC_PMI_CRAY_INCLUDE_DIRS} + ${PC_PMI_INCLUDEDIR} + ${PC_PMI_INCLUDE_DIRS} + PATH_SUFFIXES include +) + +find_library( + PMI_LIBRARY + NAMES pmi + HINTS ${PMI_ROOT} + ENV + PMI_ROOT + ${PC_PMI_CRAY_LIBDIR} + ${PC_PMI_CRAY_LIBRARY_DIRS} + ${PC_PMI_LIBDIR} + ${PC_PMI_LIBRARY_DIRS} + PATH_SUFFIXES lib lib64 +) + +# Set PMI_ROOT in case the other hints are used +if(PMI_ROOT) + # The call to file is for compatibility with windows paths + file(TO_CMAKE_PATH ${PMI_ROOT} PMI_ROOT) +elseif("$ENV{PMI_ROOT}") + file(TO_CMAKE_PATH $ENV{PMI_ROOT} PMI_ROOT) +else() + file(TO_CMAKE_PATH "${PMI_INCLUDE_DIR}" PMI_INCLUDE_DIR) + string(REPLACE "/include" "" PMI_ROOT "${PMI_INCLUDE_DIR}") +endif() + +if(NOT PMI_LIBRARY OR NOT PMI_INCLUDE_DIR) + set(PMI_FOUND=OFF) + return() +endif() + +# hpx_error( "PMI_LIBRARY OR PMI_INCLUDE_DIR not found, please install PMI or +# set \ the right PMI_ROOT path" ) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(PMI DEFAULT_MSG PMI_LIBRARY PMI_INCLUDE_DIR) + +mark_as_advanced(PMI_ROOT PMI_LIBRARY PMI_INCLUDE_DIR) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake new file mode 100644 index 000000000000..9c603343ef33 --- /dev/null +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -0,0 +1,1002 @@ +# Copyright (c) 2019-2023 Ste||ar Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +include(HPX_Message) + +macro(hpx_setup_openshmem) + + if(NOT TARGET PkgConfig::OPENSHMEM) + + set(OPENSHMEM_PC "") + if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") + set(OPENSHMEM_PC "ucx") + + pkg_search_module(UCX IMPORTED_TARGET GLOBAL ucx) + + if(NOT UCX_FOUND) + message( + FATAL + "HPX_WITH_PARCELPORT_OPENSHMEM=ucx selected but UCX is unavailable" + ) + endif() + + pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL osss-ucx) + elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") + set(OPENSHMEM_PC + "sandia-openshmem"") + + pkg_search_module( + OPENSHMEM IMPORTED_TARGET GLOBAL + sandia-openshmem + ) + endif() + endif() + + if(NOT OPENSHMEM_FOUND) + + find_program(MAKE_EXECUTABLE NAMES gmake make mingw32-make REQUIRED) + + include(FindOpenShmemPmi) + + set(PMI_AUTOCONF_OPTS) + if(NOT PMI_LIBRARY OR NOT PMI_FOUND) + set(PMI_AUTOCONF_OPTS --enable-pmi-simple) + else() + set(PMI_AUTOCONF_OPTS "--with-pmi=${PMI_INCLUDE_DIR} + --with-pmi-libdir=${PMI_LIBRARY}") + endif() + + include(FetchContent) + + if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") + + message(STATUS "Fetching + OSSS-UCX-OpenSHMEM") + + fetchcontent_declare( + openshmem + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + URL https://github.com/openshmem-org/osss-ucx/archive/refs/tags/v1.0.2.tar.gz + ) + + message(STATUS "Building + OSSS-UCX + (OpenSHMEM on UCX) + and + installing + into + ${CMAKE_INSTALL_PREFIX}") + + elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") + + message(STATUS "Fetching + Sandia-OpenSHMEM") + + fetchcontent_declare( + openshmem + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + URL https://github.com/Sandia-OpenSHMEM/SOS/archive/refs/tags/v1.5.2.tar.gz + ) + + message(STATUS "Building + and + installing + Sandia + OpenSHMEM + into + ${CMAKE_INSTALL_PREFIX}") + + else() + message(FATAL_ERROR "HPX_WITH_PARCELPORT_OPENSHMEM + is + not + set + to + `ucx` + or + `sos`") + endif() + + fetchcontent_getproperties(openshmem) + if(NOT openshmem) + fetchcontent_populate(openshmem) + endif() + + set(CMAKE_PREFIX_PATH "${CMAKE_INSTALL_PREFIX}/lib/pkgconfig") + set(ENV{PKG_CONFIG_PATH} "${CMAKE_INSTALL_PREFIX}/lib/pkgconfig") + + set(OPENSHMEM_DIR "${openshmem_SOURCE_DIR}") + set(OPENSHMEM_BUILD_OUTPUT "${OPENSHMEM_DIR}/build.log") + set(OPENSHMEM_ERROR_FILE "${OPENSHMEM_DIR}/error.log") + + execute_process( + COMMAND + bash -c + "CC=${CMAKE_C_COMPILER} + ./autogen.sh + && + CC=${CMAKE_C_COMPILER} + ./configure + --prefix=${OPENSHMEM_DIR}/install + --enable-shared + ${PMI_AUTOCONF_OPTS} + && + make + && + make + install" + WORKING_DIRECTORY ${OPENSHMEM_DIR} + RESULT_VARIABLE OPENSHMEM_BUILD_STATUS + OUTPUT_FILE ${OPENSHMEM_BUILD_OUTPUT} + ERROR_FILE ${OPENSHMEM_ERROR_FILE} + ) + + if(OPENSHMEM_BUILD_STATUS) + message(FATAL_ERROR "OpenSHMEM + build + result + = + ${OPENSHMEM_SRC_BUILD_STATUS} + - + see + ${OPENSHMEM_SRC_BUILD_OUTPUT} + for + more + details") + else() + + find_file(OPENSHMEM_PKGCONFIG_FILE_FOUND + ${OPENSHMEM_PC} + ${OPENSHMEM_DIR}/install/lib/pkgconfig + ) + + if(NOT OPENSHMEM_PKGCONFIG_FILE_FOUND) + message( + FATAL_ERROR + "PKG-CONFIG + ERROR + (${OPENSHMEM_PKGCONFIG_FILE_FOUND}) + -> + CANNOT + FIND + COMPILED + OpenSHMEM: + ${OPENSHMEMT_DIR}/install/lib/pkgconfig" + ) + endif() + + install(CODE "set + (OPENSHMEM_PATH \"${OPENSHMEM_DIR}\") + ") + + install( + CODE [[ + file( + READ + ${OPENSHMEM_PATH}/install/lib/pkgconfig/${OPENSHMEM_PC} + OPENSHMEM_PKGCONFIG_FILE_CONTENT + ) + + if(NOT OPENSHMEM_PKGCONFIG_FILE_CONTENT) + message(FATAL_ERROR "ERROR + INSTALLING + OPENSHMEM") + endif() + + string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" + OPENSHMEM_PKGCONFIG_FILE_CONTENT + ${OPENSHMEM_PKGCONFIG_FILE_CONTENT} + ) + + file( + WRITE + ${OPENSHMEM_PATH}/install/lib/pkgconfig/${OPENSHMEM_PC} + ${OPENSHMEM_PKGCONFIG_FILE_CONTENT} + ) + + file(GLOB_RECURSE OPENSHMEM_FILES ${OPENSHMEM_PATH}/install/*) + + if(NOT OPENSHMEM_FILES) + message(STATUS "ERROR + INSTALLING + OPENSHMEM") + endif() + + foreach(OPENSHMEM_FILE ${OPENSHMEM_FILES}) + set(OPENSHMEM_FILE_CACHED "${OPENSHMEM_FILE}") + + string(REGEX MATCH " + (^ \/.*\/) + " OPENSHMEM_FILE_PATH ${OPENSHMEM_FILE}) + + string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" + OPENSHMEM_FILE ${OPENSHMEM_FILE} + ) + + string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" + OPENSHMEM_FILE_PATH ${OPENSHMEM_FILE_PATH} + ) + + file(MAKE_DIRECTORY ${OPENSHMEM_FILE_PATH}) + + string(LENGTH ${OPENSHMEM_FILE_PATH} OPENSHMEM_FILE_PATH_SIZE) + math(EXPR OPENSHMEM_FILE_PATH_SIZE "${OPENSHMEM_FILE_PATH_SIZE}-1") + + string(SUBSTRING ${OPENSHMEM_FILE_PATH} 0 ${OPENSHMEM_FILE_PATH_SIZE} + OPENSHMEM_FILE_PATH + ) + + file(COPY ${OPENSHMEM_FILE_CACHED} DESTINATION ${OPENSHMEM_FILE_PATH}) + endforeach() + ]] + ) + + # install(FILES ${OPENSHMEM_FILES} DESTINATION ${CMAKE_INSTALL_PREFIX}) + endif() + + set(CMAKE_PREFIX_PATH "${OPENSHMEM_DIR}/install/lib/pkgconfig") + set(ENV{PKG_CONFIG_PATH} "${OPENSHMEM_DIR}/install/lib/pkgconfig") + + if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") + pkg_search_module( + OPENSHMEM IMPORTED_TARGET GLOBAL + osss-ucx + ) + elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") + pkg_search_module( + OPENSHMEM IMPORTED_TARGET GLOBAL + sandia-openshmem + ) + endif() + + if(NOT OPENSHMEM_FOUND) + message(FATAL_ERROR "OpenSHMEM + downloaded, + compiled, + but + cannot + be + found + in + ${CMAKE_INSTALL_PREFIX}") + endif() + + endif() + + if(OPENSHMEM_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH OPENSHMEM_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_CFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_CFLAGS "${X}") + endforeach() + endif() + + if(OPENSHMEM_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH OPENSHMEM_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_CFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_CFLAGS_OTHER "${X}") + endforeach() + endif() + + if(OPENSHMEM_LDFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(SKIP 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lsma" IDX) + string(FIND "${X}" "-l" LIDX) + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endif() + endforeach() + + set(IDX 0) + list(LENGTH OPENSHMEM_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " + -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + endif() + + if(OPENSHMEM_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(SKIP 0) + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lsma" IDX) + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endif() + endforeach() + + set(IDX 0) + list(LENGTH OPENSHMEM_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_LDFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_LDFLAGS_OTHER "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " + -Wl,--no-whole-archive") + + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS_OTHER "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + + endif() + + if(OPENSHMEM_STATIC_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_STATIC_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH OPENSHMEM_STATIC_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_STATIC_CFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_STATIC_CFLAGS "${X}") + endforeach() + endif() + + if(OPENSHMEM_STATIC_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_STATIC_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() + + list(LENGTH OPENSHMEM_STATIC_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_STATIC_CFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_STATIC_CFLAGS_OTHER "${X}") + endforeach() + endif() + + if(OPENSHMEM_STATIC_LDFLAGS) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(SKIP 0) + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_STATIC_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lsma" IDX) + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endif() + endforeach() + + set(IDX 0) + list(LENGTH OPENSHMEM_STATIC_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_STATIC_LDFLAGS NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_STATIC_LDFLAGS "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " + -Wl,--no-whole-archive") + + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_STATIC_LDFLAGS "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + endif() + + if(OPENSHMEM_STATIC_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(SKIP 0) + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_STATIC_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lsma" IDX) + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IDX}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}") + set(NEWPARAM "") + set(IS_PARAM "0") + elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-l" "" TMPSTR "${X}") + list(APPEND LIB_LIST "${TMPSTR}") + set(IDX 0) + elseif("${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") + list(APPEND FLAG_LIST "${X}") + endif() + if(NOT "${DIRIDX}" EQUAL "-1") + set(TMPSTR "") + string(REPLACE "-L" "" TMPSTR "${X}") + list(APPEND DIR_LIST "${TMPSTR}") + endif() + endif() + endforeach() + + set(IDX 0) + list(LENGTH OPENSHMEM_STATIC_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_STATIC_LDFLAGS_OTHER NEWPARAM) + endforeach() + + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_STATIC_LDFLAGS_OTHER "${X}") + endforeach() + + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") + set(IDX 0) + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + ") + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + message(STATUS "${FOUND_LIB} + ${X}") + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(APPEND NEWLINK " + -Wl,--no-whole-archive") + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_STATIC_LDFLAGS_OTHER "${NEWLINK}") + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + ") + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}") + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" + "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") + endif() + endif() + endif() + endif() + + if(OPENSHMEM_DIR) + list(TRANSFORM OPENSHMEM_CFLAGS + REPLACE "${OPENSHMEM_DIR}/install" + "$" + ) + list(TRANSFORM OPENSHMEM_LDFLAGS + REPLACE "${OPENSHMEM_DIR}/install" + "$" + ) + list(TRANSFORM OPENSHMEM_LIBRARY_DIRS + REPLACE "${OPENSHMEM_DIR}/install" + "$" + ) + + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS + "${OPENSHMEM_CFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS + "${OPENSHMEM_LDFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES + "${OPENSHMEM_LIBRARY_DIRS}" + ) + else() + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS "${OPENSHMEM_CFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS "${OPENSHMEM_LDFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES + "${OPENSHMEM_LIBRARY_DIRS} + " + ) + endif() + + endif() +endmacro() diff --git a/cmake/templates/HPXConfig.cmake.in b/cmake/templates/HPXConfig.cmake.in index c3f0ae31b8f7..156240aebcf6 100644 --- a/cmake/templates/HPXConfig.cmake.in +++ b/cmake/templates/HPXConfig.cmake.in @@ -52,6 +52,11 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) endif() endif() +if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_OPENSHMEM) + include(HPX_SetupOpenSHMEM) + hpx_setup_openshmem() +endif() + if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET) include(HPX_SetupGasnet) hpx_setup_gasnet() diff --git a/cmake/templates/hpxrun.py.in b/cmake/templates/hpxrun.py.in index 21a5badbd411..e5de23a6a101 100755 --- a/cmake/templates/hpxrun.py.in +++ b/cmake/templates/hpxrun.py.in @@ -203,6 +203,14 @@ def run_gasnet(cmd, localities, verbose): print('Executing command: ' + ' '.join(exec_cmd)) subproc(exec_cmd) +# Run with oshrun +# This is executing amudprun with the "-np" option set to the number of localities +def run_openshmem(cmd, localities, verbose): + exec_cmd = ['oshrun', '-np', str(localities)] + cmd + if verbose: + print('Executing command: ' + ' '.join(exec_cmd)) + subproc(exec_cmd) + # Run with srun # This is executing srun with the '-n' option set to the number of localities def run_srun(cmd, localities, verbose): @@ -229,6 +237,9 @@ def run(cmd, runwrapper, localities, nodes, verbose): if runwrapper == 'gasnet': assert nodes is None, "nodes option only valid with tcp parcelport." run_gasnet(cmd, localities, verbose) + if runwrapper == 'openshmem': + assert nodes is None, "nodes option only valid with tcp parcelport." + run_openshmem(cmd, localities, verbose) if runwrapper == 'srun': assert nodes is None, "nodes option only valid with tcp parcelport." run_srun(cmd, localities, verbose) diff --git a/libs/core/CMakeLists.txt b/libs/core/CMakeLists.txt index 3e6550e93f25..a55d1eaac8e9 100644 --- a/libs/core/CMakeLists.txt +++ b/libs/core/CMakeLists.txt @@ -56,6 +56,7 @@ set(_hpx_core_modules logging memory mpi_base + openshmem_base pack_traversal plugin prefix diff --git a/libs/core/openshmem_base/CMakeLists.txt b/libs/core/openshmem_base/CMakeLists.txt new file mode 100644 index 000000000000..c6164fcfa68e --- /dev/null +++ b/libs/core/openshmem_base/CMakeLists.txt @@ -0,0 +1,31 @@ +# Copyright (c) 2023 Christopher Taylor +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +if(NOT (HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_OPENSHMEM)) + return() +endif() + +include(HPX_SetupOpenSHMEM) +hpx_setup_openshmem() + +# Default location is $HPX_ROOT/libs/openshmem_base/include +set(openshmem_base_headers hpx/openshmem_base/openshmem.hpp + hpx/openshmem_base/openshmem_environment.hpp +) + +set(openshmem_base_sources openshmem_environment.cpp) + +include(HPX_AddModule) +add_hpx_module( + core openshmem_base + GLOBAL_HEADER_GEN ON + SOURCES ${openshmem_base_sources} + HEADERS ${openshmem_base_headers} + DEPENDENCIES PkgConfig::OPENSHMEM + MODULE_DEPENDENCIES hpx_logging hpx_runtime_configuration hpx_string_util + hpx_threading_base hpx_util + CMAKE_SUBDIRS examples tests +) diff --git a/libs/core/openshmem_base/examples/CMakeLists.txt b/libs/core/openshmem_base/examples/CMakeLists.txt new file mode 100644 index 000000000000..ff6189b08a6f --- /dev/null +++ b/libs/core/openshmem_base/examples/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright (c) 2020 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +if(HPX_WITH_EXAMPLES) + add_hpx_pseudo_target(examples.modules.openshmem_base) + add_hpx_pseudo_dependencies(examples.modules examples.modules.openshmem_base) + if(HPX_WITH_TESTS AND HPX_WITH_TESTS_EXAMPLES) + add_hpx_pseudo_target(tests.examples.modules.openshmem_base) + add_hpx_pseudo_dependencies( + tests.examples.modules tests.examples.modules.openshmem_base + ) + endif() +endif() diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp new file mode 100644 index 000000000000..e3f6bcb3a94f --- /dev/null +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem.hpp @@ -0,0 +1,36 @@ +// Copyright (c) 2023 Christoper Taylor +// Copyright (c) 2017 Mikael Simberg +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) + +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#ifdef __cplusplus +} // extern "C" +#endif + +#if defined(__clang__) +#pragma clang diagnostic pop +#elif defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + +#endif diff --git a/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp new file mode 100644 index 000000000000..af57d81b0fe4 --- /dev/null +++ b/libs/core/openshmem_base/include/hpx/openshmem_base/openshmem_environment.hpp @@ -0,0 +1,126 @@ +// Copyright (c) 2013-2015 Thomas Heller +// Copyright (c) 2023 Christopher Taylor +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +#if !defined(OPENSHMEM_PAGESIZE) +// GASNet default pagesize is 16384 +#define OPENSHMEM_PER_RANK_PAGESIZE 16384 +#endif + +namespace hpx { namespace util { + + struct openshmem_seginfo_t + { + std::uint8_t* addr; + std::uint8_t* size; + }; + + struct HPX_CORE_EXPORT openshmem_environment + { + static bool check_openshmem_environment( + runtime_configuration const& cfg); + + static int init(int* argc, char*** argv, int& provided); + static void init(int* argc, char*** argv, runtime_configuration& cfg); + static void finalize(); + + static bool enabled(); + static bool multi_threaded(); + static bool has_called_init(); + + static int rank(); + static int size(); + + static std::string get_processor_name(); + + static void put_signal(const std::uint8_t* addr, const int rank, + std::uint8_t* raddr, const std::size_t size, unsigned int* sigaddr); + + static void wait_until(const unsigned int value, unsigned int* sigaddr); + + static void get(std::uint8_t* addr, const int rank, + const std::uint8_t* raddr, const std::size_t size); + + static void global_barrier(); + + struct HPX_CORE_EXPORT scoped_lock + { + scoped_lock(); + scoped_lock(scoped_lock const&) = delete; + scoped_lock& operator=(scoped_lock const&) = delete; + ~scoped_lock(); + void unlock(); + }; + + struct HPX_CORE_EXPORT scoped_try_lock + { + scoped_try_lock(); + scoped_try_lock(scoped_try_lock const&) = delete; + scoped_try_lock& operator=(scoped_try_lock const&) = delete; + ~scoped_try_lock(); + void unlock(); + bool locked; + }; + + typedef hpx::spinlock mutex_type; + + public: + static hpx::spinlock pollingLock; + static hpx::mutex mtx_; + + static bool enabled_; + static bool has_called_init_; + static int provided_threading_flag_; + + static int is_initialized_; + + static hpx::mutex dshm_mut; + static int init_val_; + static hpx::mutex* segment_mutex; + static openshmem_seginfo_t* segments; + static std::uint8_t* shmem_buffer; + static unsigned int rcv; + static unsigned int xmt; + }; +}} // namespace hpx::util + +#include + +#else + +#include + +#include + +namespace hpx { namespace util { + struct HPX_CORE_EXPORT openshmem_environment + { + static bool check_openshmem_environment( + runtime_configuration const& cfg); + }; +}} // namespace hpx::util + +#include + +#endif diff --git a/libs/core/openshmem_base/src/openshmem_environment.cpp b/libs/core/openshmem_base/src/openshmem_environment.cpp new file mode 100644 index 000000000000..1274c201c487 --- /dev/null +++ b/libs/core/openshmem_base/src/openshmem_environment.cpp @@ -0,0 +1,378 @@ +// Copyright (c) 2013-2015 Thomas Heller +// Copyright (c) 2020 Google +// Copyright (c) 2022 Patrick Diehl +// Copyright (c) 2023 Christopher Taylor +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +/////////////////////////////////////////////////////////////////////////////// +namespace hpx::util { + + namespace detail { + + bool detect_openshmem_environment( + util::runtime_configuration const& cfg, char const* default_env) + { + std::string openshmem_environment_strings = + cfg.get_entry("hpx.parcel.openshmem.env", default_env); + + hpx::string_util::char_separator sep(";,: "); + hpx::string_util::tokenizer tokens( + openshmem_environment_strings, sep); + for (auto const& tok : tokens) + { + char* env = std::getenv(tok.c_str()); + if (env) + { + LBT_(debug) + << "Found OPENSHMEM environment variable: " << tok + << "=" << std::string(env) + << ", enabling OPENSHMEM support\n"; + return true; + } + } + + LBT_(info) + << "No known OPENSHMEM environment variable found, disabling " + "OPENSHMEM support\n"; + return false; + } + } // namespace detail + + bool openshmem_environment::check_openshmem_environment( + util::runtime_configuration const& cfg) + { +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) + // We disable the OPENSHMEM parcelport if any of these hold: + // + // - The parcelport is explicitly disabled + // - The application is not run in an OPENSHMEM environment + // - The TCP parcelport is enabled and has higher priority + // + if (get_entry_as(cfg, "hpx.parcel.openshmem.enable", 1) == 0 || + (get_entry_as(cfg, "hpx.parcel.tcp.enable", 1) && + (get_entry_as(cfg, "hpx.parcel.tcp.priority", 1) > + get_entry_as(cfg, "hpx.parcel.openshmem.priority", 0))) || + (get_entry_as(cfg, "hpx.parcel.openshmem.enable", 1) && + (get_entry_as(cfg, "hpx.parcel.openshmem.priority", 1) > + get_entry_as(cfg, "hpx.parcel.mpi.priority", 0)))) + { + LBT_(info) + << "OpenSHMEM support disabled via configuration settings\n"; + return false; + } + + return true; +#else + return false; +#endif + } +} // namespace hpx::util + +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_MODULE_OPENSHMEM_BASE)) + +namespace hpx::util { + + hpx::spinlock openshmem_environment::pollingLock{}; + hpx::mutex openshmem_environment::dshm_mut{}; + hpx::mutex openshmem_environment::mtx_{}; + bool openshmem_environment::enabled_ = false; + bool openshmem_environment::has_called_init_ = false; + int openshmem_environment::provided_threading_flag_ = 0; + int openshmem_environment::is_initialized_ = -1; + int openshmem_environment::init_val_ = 0; + hpx::mutex* openshmem_environment::segment_mutex = nullptr; + openshmem_seginfo_t* openshmem_environment::segments = nullptr; + std::uint8_t* hpx::util::openshmem_environment::shmem_buffer = nullptr; + unsigned int openshmem_environment::rcv = 0; + unsigned int openshmem_environment::xmt = 0; + + /////////////////////////////////////////////////////////////////////////// + int openshmem_environment::init(int* argc, char*** argv, int& provided) + { + if (!has_called_init_) + { + shmem_init(); + openshmem_environment::init_val_ = 1; + has_called_init_ = true; + } + + if (openshmem_environment::init_val_ == 0) + { + HPX_THROW_EXCEPTION(error::invalid_status, + "hpx::util::openshmem_environment::init", + "OPENSHMEM initialization error"); + } + else if (openshmem_environment::init_val_ == 0) + { + HPX_THROW_EXCEPTION(error::invalid_status, + "hpx::util::openshmem_environment::init", + "OPENSHMEM resource error"); + } + else if (openshmem_environment::init_val_ == 0) + { + HPX_THROW_EXCEPTION(error::invalid_status, + "hpx::util::openshmem_environment::init", + "OPENSHMEM bad argument error"); + } + else if (openshmem_environment::init_val_ == 0) + { + HPX_THROW_EXCEPTION(error::invalid_status, + "hpx::util::openshmem_environment::init", + "OPENSHMEM not ready error"); + } + + // create a number of segments equal to the number of hardware + // threads per machine (locality) + // + //segments.resize(hpx::threads::hardware_concurrency() * size()); + // + openshmem_environment::segments = new openshmem_seginfo_t[size()]; + openshmem_environment::segment_mutex = new hpx::mutex[size()]; + + hpx::util::openshmem_environment::shmem_buffer = + static_cast(shmem_calloc( + OPENSHMEM_PER_RANK_PAGESIZE, sizeof(std::uint8_t))); + + for (int i = 0; i < size(); ++i) + { + segments[i].addr = hpx::util::openshmem_environment::shmem_buffer + + (i * OPENSHMEM_PER_RANK_PAGESIZE); + segments[i].size = static_cast(segments[i].addr) + + OPENSHMEM_PER_RANK_PAGESIZE; + } + + shmem_barrier_all(); + + return openshmem_environment::init_val_; + } + + /////////////////////////////////////////////////////////////////////////// + void openshmem_environment::init( + int* argc, char*** argv, util::runtime_configuration& rtcfg) + { + if (enabled_) + return; // don't call twice + + int this_rank = -1; + has_called_init_ = false; + + // We assume to use the OpenSHMEM parcelport if it is not explicitly disabled + enabled_ = check_openshmem_environment(rtcfg); + if (!enabled_) + { + rtcfg.add_entry("hpx.parcel.openshmem.enable", "0"); + return; + } + + rtcfg.add_entry("hpx.parcel.bootstrap", "openshmem"); + + int retval = init(argc, argv, provided_threading_flag_); + if (1 != retval) + { + // explicitly disable openshmem if not run by openshmemrun + rtcfg.add_entry("hpx.parcel.openshmem.enable", "0"); + + enabled_ = false; + + std::string msg( + "openshmem_environment::init: openshmem_init failed"); + throw std::runtime_error(msg.c_str()); + } + + if (provided_threading_flag_ != 1) + { + // explicitly disable openshmem if not run by openshmemrun + rtcfg.add_entry("hpx.parcel.openshmem.multithreaded", "0"); + } + + this_rank = rank(); + +#if defined(HPX_HAVE_NETWORKING) + if (this_rank == 0) + { + rtcfg.mode_ = hpx::runtime_mode::console; + } + else + { + rtcfg.mode_ = hpx::runtime_mode::worker; + } +#elif defined(HPX_HAVE_DISTRIBUTED_RUNTIME) + rtcfg.mode_ = hpx::runtime_mode::console; +#else + rtcfg.mode_ = hpx::runtime_mode::local; +#endif + + rtcfg.add_entry("hpx.parcel.openshmem.rank", std::to_string(this_rank)); + rtcfg.add_entry( + "hpx.parcel.openshmem.processorname", get_processor_name()); + } + + std::string openshmem_environment::get_processor_name() + { + char name[1024 + 1] = {'\0'}; + const std::string rnkstr = std::to_string(rank()); + const int len = rnkstr.size(); + if (1025 < len) + { + HPX_THROW_EXCEPTION(error::invalid_status, + "hpx::util::openshmem_environment::get_processor_name", + "openshmem processor name is larger than 1025"); + } + std::copy(std::begin(rnkstr), std::end(rnkstr), name); + return name; + } + + void openshmem_environment::put_signal(const std::uint8_t* addr, + const int node, std::uint8_t* raddr, const std::size_t size, + unsigned int* sigaddr) + { + if (rank() == node) + { + const std::lock_guard lk(segment_mutex[node]); + std::memmove(raddr, addr, size); + } + else + { + const std::lock_guard lk(segment_mutex[node]); + shmem_uint8_put_signal(raddr, addr, size, + reinterpret_cast(sigaddr), 1, SHMEM_SIGNAL_SET, + node); + } + } + + void openshmem_environment::wait_until( + const unsigned int value, unsigned int* sigaddr) + { + shmem_uint_wait_until(sigaddr, SHMEM_CMP_EQ, value); + } + + void openshmem_environment::get(std::uint8_t* addr, const int node, + const std::uint8_t* raddr, const std::size_t size) + { + if (rank() == node) + { + std::memmove(addr, raddr, size); + } + else + { + shmem_uint8_get( + addr, raddr, size, node); // dest, node, src, size + } + } + + void openshmem_environment::global_barrier() + { + shmem_barrier_all(); + } + + void openshmem_environment::finalize() + { + if (enabled() && has_called_init()) + { + shmem_finalize(); + + delete segments; + delete segment_mutex; + shmem_free(hpx::util::openshmem_environment::shmem_buffer); + segments = nullptr; + segment_mutex = nullptr; + shmem_buffer = nullptr; + } + } + + bool openshmem_environment::enabled() + { + return enabled_; + } + + bool openshmem_environment::multi_threaded() + { + return provided_threading_flag_ != 0; + } + + bool openshmem_environment::has_called_init() + { + return has_called_init_; + } + + int openshmem_environment::size() + { + int res(-1); + if (enabled()) + res = static_cast(shmem_n_pes()); + return res; + } + + int openshmem_environment::rank() + { + int res(-1); + if (enabled()) + res = static_cast(shmem_my_pe()); + return res; + } + + openshmem_environment::scoped_lock::scoped_lock() + { + if (!multi_threaded()) + mtx_.lock(); + } + + openshmem_environment::scoped_lock::~scoped_lock() + { + if (!multi_threaded()) + mtx_.unlock(); + } + + void openshmem_environment::scoped_lock::unlock() + { + if (!multi_threaded()) + mtx_.unlock(); + } + + openshmem_environment::scoped_try_lock::scoped_try_lock() + : locked(true) + { + if (!multi_threaded()) + { + locked = mtx_.try_lock(); + } + } + + openshmem_environment::scoped_try_lock::~scoped_try_lock() + { + if (!multi_threaded() && locked) + mtx_.unlock(); + } + + void openshmem_environment::scoped_try_lock::unlock() + { + if (!multi_threaded() && locked) + { + locked = false; + mtx_.unlock(); + } + } +} // namespace hpx::util + +#endif diff --git a/libs/core/openshmem_base/tests/CMakeLists.txt b/libs/core/openshmem_base/tests/CMakeLists.txt new file mode 100644 index 000000000000..76ce5efbb376 --- /dev/null +++ b/libs/core/openshmem_base/tests/CMakeLists.txt @@ -0,0 +1,44 @@ +# Copyright (c) 2020 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +include(HPX_Message) +include(HPX_Option) + +if(HPX_WITH_TESTS) + if(HPX_WITH_TESTS_UNIT) + add_hpx_pseudo_target(tests.unit.modules.openshmem_base) + add_hpx_pseudo_dependencies( + tests.unit.modules tests.unit.modules.openshmem_base + ) + add_subdirectory(unit) + endif() + + if(HPX_WITH_TESTS_REGRESSIONS) + add_hpx_pseudo_target(tests.regressions.modules.openshmem_base) + add_hpx_pseudo_dependencies( + tests.regressions.modules tests.regressions.modules.openshmem_base + ) + add_subdirectory(regressions) + endif() + + if(HPX_WITH_TESTS_BENCHMARKS) + add_hpx_pseudo_target(tests.performance.modules.openshmem_base) + add_hpx_pseudo_dependencies( + tests.performance.modules tests.performance.modules.openshmem_base + ) + add_subdirectory(performance) + endif() + + if(HPX_WITH_TESTS_HEADERS) + add_hpx_header_tests( + modules.openshmem_base + HEADERS ${openshmem_base_headers} + HEADER_ROOT ${PROJECT_SOURCE_DIR}/include + NOLIBS + DEPENDENCIES hpx_openshmem_base + ) + endif() +endif() diff --git a/libs/core/openshmem_base/tests/performance/CMakeLists.txt b/libs/core/openshmem_base/tests/performance/CMakeLists.txt new file mode 100644 index 000000000000..7978545c25c2 --- /dev/null +++ b/libs/core/openshmem_base/tests/performance/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/core/openshmem_base/tests/regressions/CMakeLists.txt b/libs/core/openshmem_base/tests/regressions/CMakeLists.txt new file mode 100644 index 000000000000..7978545c25c2 --- /dev/null +++ b/libs/core/openshmem_base/tests/regressions/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/core/openshmem_base/tests/unit/CMakeLists.txt b/libs/core/openshmem_base/tests/unit/CMakeLists.txt new file mode 100644 index 000000000000..7978545c25c2 --- /dev/null +++ b/libs/core/openshmem_base/tests/unit/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/core/version/CMakeLists.txt b/libs/core/version/CMakeLists.txt index 5a82861e5ec2..f0886e68d6e4 100644 --- a/libs/core/version/CMakeLists.txt +++ b/libs/core/version/CMakeLists.txt @@ -20,6 +20,13 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) hpx_setup_lci() set(additional_dependencies ${additional_dependencies} LCI::LCI) endif() +if(HPX_WITH_NETWORKING + AND HPX_WITH_PARCELPORT_OPENSHMEM +) + include(HPX_SetupOpenSHMEM) + hpx_setup_openshmem() + set(additional_dependencies ${additional_dependencies} PkgConfig::OPENSHMEM) +endif() if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_GASNET AND HPX_WITH_PARCELPORT_GASNET_CONDUIT diff --git a/libs/core/version/src/version.cpp b/libs/core/version/src/version.cpp index 96d1662efdf2..0a48cdc99c08 100644 --- a/libs/core/version/src/version.cpp +++ b/libs/core/version/src/version.cpp @@ -46,6 +46,14 @@ #include #endif +#if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) +#include + +#define OPENSHMEM_CONDUIT_NAME_STR_HELPER(x) #x +#define OPENSHMEM_CONDUIT_NAME_STR \ + OPENSHMEM_CONDUIT_NAME_STR_HELPER(HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT) +#endif + #if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) #include @@ -136,6 +144,25 @@ namespace hpx { } #endif +#if (defined(HPX_HAVE_NETWORKING) && \ + defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) || \ + defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) + std::string openshmem_version() + { + int major = 0; + int minor = 0; + shmem_info_get_version(&major, &minor); + char vendor_cstr[SHMEM_MAX_NAME_LEN]; + shmem_info_get_name(vendor_cstr); + + std::ostringstream strm; + strm << "OPENSHMEM_VENDOR:" << vendor_cstr << ':' << major << ':' + << minor << '-' + << "OPENSHMEM_CONDUIT:" << OPENSHMEM_CONDUIT_NAME_STR; + return strm.str(); + } +#endif + #if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) std::string gasnet_version() @@ -312,6 +339,11 @@ namespace hpx { defined(HPX_HAVE_MODULE_LCI_BASE) " LCI: {}\n" #endif +#if (defined(HPX_HAVE_NETWORKING) && \ + defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) || \ + defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) + " OPENSHMEM: {}\n" +#endif #if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) " GASNET: {}\n" @@ -332,6 +364,11 @@ namespace hpx { defined(HPX_HAVE_MODULE_LCI_BASE) lci_version(), #endif +#if (defined(HPX_HAVE_NETWORKING) && \ + defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) || \ + defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) + openshmem_version(), +#endif #if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) gasnet_version(), diff --git a/libs/full/CMakeLists.txt b/libs/full/CMakeLists.txt index e4a7eb24b923..d4f1b25c0e69 100644 --- a/libs/full/CMakeLists.txt +++ b/libs/full/CMakeLists.txt @@ -35,6 +35,7 @@ set(_hpx_full_modules parcelport_lci parcelport_libfabric parcelport_mpi + parcelport_openshmem parcelport_tcp parcelports parcelset diff --git a/libs/full/command_line_handling/CMakeLists.txt b/libs/full/command_line_handling/CMakeLists.txt index 966ba221406e..cf4cd5d16212 100644 --- a/libs/full/command_line_handling/CMakeLists.txt +++ b/libs/full/command_line_handling/CMakeLists.txt @@ -34,6 +34,11 @@ if(HPX_WITH_NETWORKING AND HPX_WITH_PARCELPORT_LCI) hpx_setup_lci() set(additional_dependencies ${additional_dependencies} LCI::LCI) endif() +if(HPX_WITH_PARCELPORT_OPENSHMEM) + include(HPX_SetupOpenSHMEM) + hpx_setup_openshmem() + set(additional_dependencies ${additional_dependencies} PkgConfig::OPENSHMEM) +endif() if(HPX_WITH_PARCELPORT_GASNET) include(HPX_SetupGasnet) hpx_setup_gasnet() diff --git a/libs/full/command_line_handling/src/command_line_handling.cpp b/libs/full/command_line_handling/src/command_line_handling.cpp index 34437c3e8534..7e06038b9a27 100644 --- a/libs/full/command_line_handling/src/command_line_handling.cpp +++ b/libs/full/command_line_handling/src/command_line_handling.cpp @@ -21,6 +21,9 @@ #if defined(HPX_HAVE_MODULE_LCI_BASE) #include #endif +#if defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) +#include +#endif #if defined(HPX_HAVE_MODULE_GASNET_BASE) #include #endif @@ -999,6 +1002,18 @@ namespace hpx::util { node_ = static_cast(util::lci_environment::rank()); } #endif +#if (defined(HPX_HAVE_NETWORKING) && \ + defined(HPX_HAVE_PARCELPORT_OPENSHMEM)) || \ + defined(HPX_HAVE_MODULE_OPENSHMEM_BASE) + if (util::openshmem_environment::check_openshmem_environment(rtcfg_)) + { + util::openshmem_environment::init(&argc, &argv, rtcfg_); + num_localities_ = + static_cast(util::openshmem_environment::size()); + node_ = + static_cast(util::openshmem_environment::rank()); + } +#endif #if (defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_GASNET)) || \ defined(HPX_HAVE_MODULE_GASNET_BASE) // better to put GASNET init after MPI init, since GASNET will also diff --git a/libs/full/parcelport_openshmem/CMakeLists.txt b/libs/full/parcelport_openshmem/CMakeLists.txt new file mode 100644 index 000000000000..78575d783503 --- /dev/null +++ b/libs/full/parcelport_openshmem/CMakeLists.txt @@ -0,0 +1,45 @@ +# Copyright (c) 2023 Chistopher Taylor +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +if(NOT + (HPX_WITH_NETWORKING + AND HPX_WITH_PARCELPORT_OPENSHMEM + AND HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT) +) + return() +endif() + +include(HPX_SetupOpenSHMEM) +hpx_setup_openshmem() + +set(parcelport_openshmem_headers + hpx/parcelport_openshmem/header.hpp + hpx/parcelport_openshmem/locality.hpp + hpx/parcelport_openshmem/receiver.hpp + hpx/parcelport_openshmem/receiver_connection.hpp + hpx/parcelport_openshmem/sender.hpp + hpx/parcelport_openshmem/sender_connection.hpp + hpx/parcelport_openshmem/tag_provider.hpp +) + +set(parcelport_openshmem_sources locality.cpp parcelport_openshmem.cpp) + +include(HPX_AddModule) +add_hpx_module( + full parcelport_openshmem + GLOBAL_HEADER_GEN ON + SOURCES ${parcelport_openshmem_sources} + HEADERS ${parcelport_openshmem_headers} + DEPENDENCIES hpx_core hpx_openshmem_base PkgConfig::OPENSHMEM + ${openshmem_additional_dependencies} + MODULE_DEPENDENCIES hpx_actions hpx_command_line_handling hpx_parcelset + CMAKE_SUBDIRS examples tests +) + +set(HPX_STATIC_PARCELPORT_PLUGINS + ${HPX_STATIC_PARCELPORT_PLUGINS} parcelport_openshmem + CACHE INTERNAL "" FORCE +) diff --git a/libs/full/parcelport_openshmem/examples/CMakeLists.txt b/libs/full/parcelport_openshmem/examples/CMakeLists.txt new file mode 100644 index 000000000000..d14c5d1103c2 --- /dev/null +++ b/libs/full/parcelport_openshmem/examples/CMakeLists.txt @@ -0,0 +1,18 @@ +# Copyright (c) 2020-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +if(HPX_WITH_EXAMPLES) + add_hpx_pseudo_target(examples.modules.parcelport_openshmem) + add_hpx_pseudo_dependencies( + examples.modules examples.modules.parcelport_openshmem + ) + if(HPX_WITH_TESTS AND HPX_WITH_TESTS_EXAMPLES) + add_hpx_pseudo_target(tests.examples.modules.parcelport_openshmem) + add_hpx_pseudo_dependencies( + tests.examples.modules tests.examples.modules.parcelport_openshmem + ) + endif() +endif() diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp new file mode 100644 index 000000000000..3712581d08e2 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/header.hpp @@ -0,0 +1,150 @@ +// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2013-2021 Hartmut Kaiser +// Copyright (c) 2013-2015 Thomas Heller +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include + +#include + +#include +#include +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + struct header + { + using value_type = int; + + enum data_pos + { + pos_tag = 0 * sizeof(value_type), + pos_size = 1 * sizeof(value_type), + pos_numbytes = 2 * sizeof(value_type), + pos_numchunks_first = 3 * sizeof(value_type), + pos_numchunks_second = 4 * sizeof(value_type), + pos_piggy_back_flag = 5 * sizeof(value_type), + pos_piggy_back_data = 5 * sizeof(value_type) + 1 + }; + + static constexpr int data_size_ = 512; + + template + header(Buffer const& buffer, int tag) noexcept + { + std::int64_t size = static_cast(buffer.size_); + std::int64_t numbytes = + static_cast(buffer.data_size_); + + HPX_ASSERT(size <= (std::numeric_limits::max)()); + HPX_ASSERT(numbytes <= (std::numeric_limits::max)()); + + set(tag); + set(static_cast(size)); + set(static_cast(numbytes)); + set( + static_cast(buffer.num_chunks_.first)); + set( + static_cast(buffer.num_chunks_.second)); + + if (buffer.data_.size() <= (data_size_ - pos_piggy_back_data)) + { + data_[pos_piggy_back_flag] = 1; + std::memcpy(&data_[pos_piggy_back_data], &buffer.data_[0], + buffer.data_.size()); + } + else + { + data_[pos_piggy_back_flag] = 0; + } + } + + header() noexcept + { + reset(); + } + + void reset() noexcept + { + std::memset(&data_[0], -1, data_size_); + data_[pos_piggy_back_flag] = 1; + } + + bool valid() const noexcept + { + return data_[0] != -1; + } + + void assert_valid() const noexcept + { + HPX_ASSERT(tag() != -1); + HPX_ASSERT(size() != -1); + HPX_ASSERT(numbytes() != -1); + HPX_ASSERT(num_chunks().first != -1); + HPX_ASSERT(num_chunks().second != -1); + } + + constexpr char* data() noexcept + { + return &data_[0]; + } + + value_type tag() const noexcept + { + return get(); + } + + value_type size() const noexcept + { + return get(); + } + + value_type numbytes() const noexcept + { + return get(); + } + + std::pair num_chunks() const noexcept + { + return std::make_pair( + get(), get()); + } + + constexpr char* piggy_back() noexcept + { + if (data_[pos_piggy_back_flag]) + return &data_[pos_piggy_back_data]; + return nullptr; + } + + private: + std::array data_; + + template + void set(T const& t) noexcept + { + std::memcpy(&data_[Pos], &t, sizeof(t)); + } + + template + value_type get() const noexcept + { + value_type res; + std::memcpy(&res, &data_[Pos], sizeof(res)); + return res; + } + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/locality.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/locality.hpp new file mode 100644 index 000000000000..d0fe8b8eecf4 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/locality.hpp @@ -0,0 +1,71 @@ +// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2013-2014 Thomas Heller +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include + +#include + +namespace hpx::parcelset::policies::openshmem { + + class locality + { + public: + constexpr locality() noexcept + : rank_(-1) + { + } + + explicit constexpr locality(std::int32_t rank) noexcept + : rank_(rank) + { + } + + constexpr std::int32_t rank() const noexcept + { + return static_cast(rank_); + } + + static constexpr const char* type() noexcept + { + return "openshmem"; + } + + explicit constexpr operator bool() const noexcept + { + return rank_ != -1; + } + + HPX_EXPORT void save(serialization::output_archive& ar) const; + HPX_EXPORT void load(serialization::input_archive& ar); + + private: + friend bool operator==( + locality const& lhs, locality const& rhs) noexcept + { + return lhs.rank_ == rhs.rank_; + } + + friend bool operator<(locality const& lhs, locality const& rhs) noexcept + { + return lhs.rank_ < rhs.rank_; + } + + friend HPX_EXPORT std::ostream& operator<<( + std::ostream& os, locality const& loc) noexcept; + + int rank_; + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp new file mode 100644 index 000000000000..2c828f54e643 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver.hpp @@ -0,0 +1,170 @@ +// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2014-2015 Thomas Heller +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + template + struct receiver + { + using header_list = std::list>; + using handles_header_type = std::set>; + using connection_type = receiver_connection; + using connection_ptr = std::shared_ptr; + using connection_list = std::deque; + + struct exp_backoff + { + int numTries; + const static int maxRetries = 10; + + void operator()(unsigned int* addr) + { + if (numTries <= maxRetries) + { + if (shmem_uint_test(addr, SHMEM_CMP_EQ, 1)) + { + return; + } + hpx::this_thread::suspend( + std::chrono::microseconds(1 << numTries)); + } + else + { + numTries = 0; + } + } + }; + + explicit constexpr receiver(Parcelport& pp) noexcept + : pp_(pp) + , bo() + { + } + + void run() noexcept + { + util::openshmem_environment::scoped_lock l; + new_header(); + } + + bool background_work() noexcept + { + // We first try to accept a new connection + connection_ptr connection = accept(); + + // If we don't have a new connection, try to handle one of the + // already accepted ones. + if (!connection) + { + std::unique_lock l(connections_mtx_, std::try_to_lock); + if (l.owns_lock() && !connections_.empty()) + { + connection = HPX_MOVE(connections_.front()); + connections_.pop_front(); + } + } + + if (connection) + { + receive_messages(HPX_MOVE(connection)); + return true; + } + + return false; + } + + void receive_messages(connection_ptr connection) noexcept + { + if (!connection->receive()) + { + std::unique_lock l(connections_mtx_); + connections_.push_back(HPX_MOVE(connection)); + } + } + + connection_ptr accept() noexcept + { + std::unique_lock l(headers_mtx_, std::try_to_lock); + if (l.owns_lock()) + { + return accept_locked(l); + } + return connection_ptr(); + } + + template + connection_ptr accept_locked(Lock& header_lock) noexcept + { + connection_ptr res; + util::openshmem_environment::scoped_try_lock l; + + if (l.locked) + { + header h = new_header(); + l.unlock(); + header_lock.unlock(); + + // remote localities 'put' into the openshmem shared + // memory segment on this machine + // + res.reset(new connection_type( + hpx::util::openshmem_environment::rank(), h, pp_)); + return res; + } + return res; + } + + header new_header() noexcept + { + header h = rcv_header_; + rcv_header_.reset(); + + while (rcv_header_.data() == 0) + { + bo(&hpx::util::openshmem_environment::rcv); + } + + hpx::util::openshmem_environment::rcv = 0; + return h; + } + + Parcelport& pp_; + + hpx::spinlock headers_mtx_; + header rcv_header_; + + hpx::spinlock handles_header_mtx_; + handles_header_type handles_header_; + + hpx::spinlock connections_mtx_; + connection_list connections_; + exp_backoff bo; + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp new file mode 100644 index 000000000000..b2166090b437 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/receiver_connection.hpp @@ -0,0 +1,269 @@ +// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2014-2015 Thomas Heller +// Copyright (c) 2007-2021 Hartmut Kaiser +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + template + struct receiver_connection + { + private: + enum connection_state + { + initialized, + rcvd_transmission_chunks, + rcvd_data, + rcvd_chunks, + sent_release_tag + }; + + using data_type = std::vector; + using buffer_type = parcel_buffer; + + public: + receiver_connection(int src, header h, Parcelport& pp) noexcept + : state_(initialized) + , src_(src) + , tag_(h.tag()) + , header_(h) + , request_ptr_(false) + , chunks_idx_(0) + , pp_(pp) + { + header_.assert_valid(); +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + parcelset::data_point& data = buffer_.data_point_; + data.time_ = timer_.elapsed_nanoseconds(); + data.bytes_ = static_cast(header_.numbytes()); +#endif + buffer_.data_.resize(static_cast(header_.size())); + buffer_.num_chunks_ = header_.num_chunks(); + } + + bool receive(std::size_t num_thread = -1) + { + switch (state_) + { + case initialized: + return receive_transmission_chunks(num_thread); + + case rcvd_transmission_chunks: + return receive_data(num_thread); + + case rcvd_data: + return receive_chunks(num_thread); + + case rcvd_chunks: + return send_release_tag(num_thread); + + case sent_release_tag: + return done(); + + default: + HPX_ASSERT(false); + } + return false; + } + + bool receive_transmission_chunks(std::size_t num_thread = -1) + { + auto self_ = hpx::util::openshmem_environment::rank(); + + // determine the size of the chunk buffer + std::size_t num_zero_copy_chunks = static_cast( + static_cast(buffer_.num_chunks_.first)); + std::size_t num_non_zero_copy_chunks = static_cast( + static_cast(buffer_.num_chunks_.second)); + buffer_.transmission_chunks_.resize( + num_zero_copy_chunks + num_non_zero_copy_chunks); + if (num_zero_copy_chunks != 0) + { + buffer_.chunks_.resize(num_zero_copy_chunks); + { + hpx::util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::wait_until( + 1, &hpx::util::openshmem_environment::rcv); + hpx::util::openshmem_environment::rcv = 0; + + hpx::util::openshmem_environment::get( + reinterpret_cast( + buffer_.transmission_chunks_.data()), + self_, + hpx::util::openshmem_environment::segments[self_].addr, + static_cast(buffer_.transmission_chunks_.size() * + sizeof(buffer_type::transmission_chunk_type))); + + request_ptr_ = true; + } + } + + state_ = rcvd_transmission_chunks; + + return receive_data(num_thread); + } + + bool receive_data(std::size_t num_thread = -1) + { + if (!request_done()) + { + return false; + } + + char* piggy_back = header_.piggy_back(); + if (piggy_back) + { + std::memcpy( + &buffer_.data_[0], piggy_back, buffer_.data_.size()); + } + else + { + auto self_ = hpx::util::openshmem_environment::rank(); + hpx::util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::wait_until( + 1, &hpx::util::openshmem_environment::rcv); + hpx::util::openshmem_environment::rcv = 0; + + hpx::util::openshmem_environment::get( + reinterpret_cast(buffer_.data_.data()), + self_, + hpx::util::openshmem_environment::segments[self_].addr, + buffer_.data_.size()); + + request_ptr_ = true; + } + + state_ = rcvd_data; + + return receive_chunks(num_thread); + } + + bool receive_chunks(std::size_t num_thread = -1) + { + while (chunks_idx_ < buffer_.chunks_.size()) + { + if (!request_done()) + { + return false; + } + + std::size_t idx = chunks_idx_++; + std::size_t chunk_size = + buffer_.transmission_chunks_[idx].second; + + data_type& c = buffer_.chunks_[idx]; + c.resize(chunk_size); + { + auto self_ = hpx::util::openshmem_environment::rank(); + hpx::util::openshmem_environment::scoped_lock l; + + hpx::util::openshmem_environment::wait_until( + 1, &hpx::util::openshmem_environment::rcv); + hpx::util::openshmem_environment::rcv = 0; + + hpx::util::openshmem_environment::get( + reinterpret_cast(c.data()), self_, + hpx::util::openshmem_environment::segments[self_].addr, + c.size()); + + hpx::util::openshmem_environment::put_signal(nullptr, src_, + nullptr, 0, &hpx::util::openshmem_environment::xmt); + + request_ptr_ = true; + } + } + + state_ = rcvd_chunks; + + return send_release_tag(num_thread); + } + + bool send_release_tag(std::size_t num_thread = -1) + { + if (!request_done()) + { + return false; + } +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + parcelset::data_point& data = buffer_.data_point_; + data.time_ = timer_.elapsed_nanoseconds() - data.time_; +#endif + { + auto self_ = hpx::util::openshmem_environment::rank(); + hpx::util::openshmem_environment::scoped_lock l; + + hpx::util::openshmem_environment::wait_until( + 1, &hpx::util::openshmem_environment::rcv); + hpx::util::openshmem_environment::rcv = 0; + + hpx::util::openshmem_environment::get( + reinterpret_cast(&tag_), self_, + hpx::util::openshmem_environment::segments[self_].addr, + sizeof(int)); + + request_ptr_ = true; + } + + decode_parcels(pp_, HPX_MOVE(buffer_), num_thread); + + state_ = sent_release_tag; + + return done(); + } + + bool done() noexcept + { + return request_done(); + } + + bool request_done() noexcept + { + hpx::util::openshmem_environment::scoped_try_lock l; + if (!l.locked) + { + return false; + } + + return request_ptr_; + } + +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + hpx::chrono::high_resolution_timer timer_; +#endif + connection_state state_; + + int src_; + int tag_; + header header_; + buffer_type buffer_; + + bool request_ptr_; + std::size_t chunks_idx_; + + Parcelport& pp_; + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp new file mode 100644 index 000000000000..b4a9bba0d479 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender.hpp @@ -0,0 +1,164 @@ +// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2014-2015 Thomas Heller +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + struct sender + { + using connection_type = sender_connection; + using connection_ptr = std::shared_ptr; + using connection_list = std::deque; + + // different versions of clang-format disagree + // clang-format off + sender() noexcept + : next_free_tag_request_(-1) + , next_free_tag_(-1) + { + } + // clang-format on + + void run() noexcept + { + hpx::util::openshmem_environment::scoped_lock l; + get_next_free_tag(); + } + + connection_ptr create_connection(int dest, parcelset::parcelport* pp) + { + return std::make_shared(this, dest, pp); + } + + void add(connection_ptr const& ptr) + { + std::unique_lock l(connections_mtx_); + connections_.push_back(ptr); + } + + int acquire_tag() noexcept + { + return tag_provider_.acquire(); + } + + void send_messages(connection_ptr connection) + { + // Check if sending has been completed.... + if (connection->send()) + { + error_code ec(throwmode::lightweight); + hpx::move_only_function + postprocess_handler; + std::swap( + postprocess_handler, connection->postprocess_handler_); + postprocess_handler(ec, connection->destination(), connection); + } + else + { + std::unique_lock l(connections_mtx_); + connections_.push_back(HPX_MOVE(connection)); + } + } + + bool background_work() noexcept + { + connection_ptr connection; + { + std::unique_lock l(connections_mtx_, std::try_to_lock); + if (l && !connections_.empty()) + { + connection = HPX_MOVE(connections_.front()); + connections_.pop_front(); + } + } + + bool has_work = false; + if (connection) + { + send_messages(HPX_MOVE(connection)); + has_work = true; + } + next_free_tag(); + return has_work; + } + + private: + tag_provider tag_provider_; + + void next_free_tag() noexcept + { + int next_free = -1; + { + std::unique_lock l(next_free_tag_mtx_, std::try_to_lock); + if (l.owns_lock()) + { + next_free = next_free_tag_locked(); + } + } + + if (next_free != -1) + { + HPX_ASSERT(next_free > 1); + tag_provider_.release(next_free); + } + } + + int next_free_tag_locked() noexcept + { + hpx::util::openshmem_environment::scoped_try_lock l; + if (l.locked) + { + return get_next_free_tag(); + } + return -1; + } + + int get_next_free_tag() noexcept + { + int next_free = next_free_tag_; + + hpx::util::openshmem_environment::scoped_lock l; + std::memcpy(&next_free, + hpx::util::openshmem_environment::segments + [hpx::util::openshmem_environment::rank()] + .addr, + sizeof(int)); + + return next_free; + } + + hpx::spinlock connections_mtx_; + connection_list connections_; + + hpx::spinlock next_free_tag_mtx_; + int next_free_tag_request_; + int next_free_tag_; + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp new file mode 100644 index 000000000000..eb1047405a61 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/sender_connection.hpp @@ -0,0 +1,325 @@ +// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2014-2015 Thomas Heller +// Copyright (c) 2023 Christopher Taylor +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + struct sender; + struct sender_connection; + + int acquire_tag(sender*) noexcept; + void add_connection(sender*, std::shared_ptr const&); + + struct sender_connection + : parcelset::parcelport_connection> + { + private: + using sender_type = sender; + + using write_handler_type = + hpx::function; + + using data_type = std::vector; + + enum connection_state + { + initialized, + sent_header, + sent_transmission_chunks, + sent_data, + sent_chunks + }; + + using base_type = + parcelset::parcelport_connection; + + public: + sender_connection(sender_type* s, int dst, parcelset::parcelport* pp) + : state_(initialized) + , sender_(s) + , tag_(-1) + , dst_(dst) + , chunks_idx_(0) + , ack_(0) + , pp_(pp) + , there_(parcelset::locality(locality(dst_))) + { + } + + parcelset::locality const& destination() const noexcept + { + return there_; + } + + constexpr void verify_( + parcelset::locality const& /* parcel_locality_id */) const noexcept + { + } + + template + void async_write( + Handler&& handler, ParcelPostprocess&& parcel_postprocess) + { + HPX_ASSERT(!handler_); + HPX_ASSERT(!postprocess_handler_); + HPX_ASSERT(!buffer_.data_.empty()); + +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + buffer_.data_point_.time_ = + hpx::chrono::high_resolution_clock::now(); +#endif + chunks_idx_ = 0; + tag_ = acquire_tag(sender_); + header_ = header(buffer_, tag_); + header_.assert_valid(); + + state_ = initialized; + + handler_ = HPX_FORWARD(Handler, handler); + + if (!send()) + { + postprocess_handler_ = + HPX_FORWARD(ParcelPostprocess, parcel_postprocess); + add_connection(sender_, shared_from_this()); + } + else + { + HPX_ASSERT(!handler_); + error_code ec; + parcel_postprocess(ec, there_, shared_from_this()); + } + } + + bool send() + { + switch (state_) + { + case initialized: + return send_header(); + + case sent_header: + return send_transmission_chunks(); + + case sent_transmission_chunks: + return send_data(); + + case sent_data: + return send_chunks(); + + case sent_chunks: + return done(); + + default: + HPX_ASSERT(false); + } + return false; + } + + bool send_header() + { + { + hpx::util::openshmem_environment::scoped_lock l; + HPX_ASSERT(state_ == initialized); + + // compute + send the number of OPENSHMEM_PAGEs to send and the + // remainder number of bytes to a OPENSHMEM_PAGE + // + std::size_t chunks[] = {static_cast(header_.data_size_ / + OPENSHMEM_PER_RANK_PAGESIZE), + static_cast( + header_.data_size_ % OPENSHMEM_PER_RANK_PAGESIZE)}; + const std::size_t sizeof_chunks = sizeof(chunks); + + // put from this localities openshmem shared memory segment + // into the remote locality (dst_)'s shared memory segment + // + hpx::util::openshmem_environment::put_signal( + reinterpret_cast(chunks), dst_, + static_cast( + hpx::util::openshmem_environment::segments[dst_].addr), + sizeof_chunks, &hpx::util::openshmem_environment::rcv); + } + + state_ = sent_header; + return send_transmission_chunks(); + } + + bool send_transmission_chunks() + { + HPX_ASSERT(state_ == sent_header); + if (!request_done()) + { + return false; + } + + std::vector& + chunks = buffer_.transmission_chunks_; + if (!chunks.empty()) + { + hpx::util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::put_signal( + reinterpret_cast(chunks.data()), dst_, + static_cast( + hpx::util::openshmem_environment::segments + [hpx::util::openshmem_environment::rank()] + .addr), + static_cast(chunks.size() * + sizeof(parcel_buffer_type::transmission_chunk_type)), + &hpx::util::openshmem_environment::rcv); + } + + state_ = sent_transmission_chunks; + return send_data(); + } + + bool send_data() + { + HPX_ASSERT(state_ == sent_transmission_chunks); + if (!request_done()) + { + return false; + } + + if (!header_.piggy_back()) + { + hpx::util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::put_signal( + reinterpret_cast(buffer_.data_.data()), dst_, + static_cast( + hpx::util::openshmem_environment::segments[dst_].addr), + buffer_.data_.size(), + &hpx::util::openshmem_environment::rcv); + } + state_ = sent_data; + + return send_chunks(); + } + + bool send_chunks() + { + HPX_ASSERT(state_ == sent_data); + + while (chunks_idx_ < buffer_.chunks_.size()) + { + serialization::serialization_chunk& c = + buffer_.chunks_[chunks_idx_]; + if (c.type_ == serialization::chunk_type::chunk_type_pointer) + { + if (!request_done()) + { + return false; + } + + hpx::util::openshmem_environment::scoped_lock l; + hpx::util::openshmem_environment::put_signal( + reinterpret_cast(c.data_.cpos_), + dst_, + static_cast( + hpx::util::openshmem_environment::segments[dst_] + .addr), + static_cast(c.size_), + &hpx::util::openshmem_environment::rcv); + + hpx::util::openshmem_environment::wait_until( + 1, &hpx::util::openshmem_environment::xmt); + hpx::util::openshmem_environment::xmt = 0; + } + + ++chunks_idx_; + } + + state_ = sent_chunks; + + return done(); + } + + bool done() + { + if (!request_done()) + { + return false; + } + + error_code ec(throwmode::lightweight); + handler_(ec); + handler_.reset(); +#if defined(HPX_HAVE_PARCELPORT_COUNTERS) + buffer_.data_point_.time_ = + hpx::chrono::high_resolution_clock::now() - + buffer_.data_point_.time_; + pp_->add_sent_data(buffer_.data_point_); +#endif + buffer_.clear(); + + state_ = initialized; + + return true; + } + + bool request_done() + { + hpx::util::openshmem_environment::scoped_try_lock l; + if (!l.locked) + { + return false; + } + + return true; + } + + connection_state state_; + sender_type* sender_; + int tag_; + int dst_; + + using handler_type = hpx::move_only_function; + handler_type handler_; + + using post_handler_type = hpx::move_only_function)>; + post_handler_type postprocess_handler_; + + header header_; + + std::size_t chunks_idx_; + char ack_; + + parcelset::parcelport* pp_; + + parcelset::locality there_; + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/tag_provider.hpp b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/tag_provider.hpp new file mode 100644 index 000000000000..6e75ac5e5482 --- /dev/null +++ b/libs/full/parcelport_openshmem/include/hpx/parcelport_openshmem/tag_provider.hpp @@ -0,0 +1,61 @@ +// Copyright (c) 2014-2015 Thomas Heller +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#pragma once + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include + +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + struct tag_provider + { + tag_provider() noexcept + : next_tag_(2) + { + } + + int acquire() noexcept + { + int tag = -1; + std::lock_guard l(mtx_); + if (free_tags_.empty()) + { + HPX_ASSERT(next_tag_ < (std::numeric_limits::max)()); + tag = next_tag_++; + } + else + { + tag = free_tags_.front(); + free_tags_.pop_front(); + } + HPX_ASSERT(tag > 1); + return tag; + } + + void release(int tag) + { + HPX_ASSERT(tag > 1); + std::lock_guard l(mtx_); + HPX_ASSERT(tag < next_tag_); + + free_tags_.push_back(tag); + } + + hpx::spinlock mtx_; + int next_tag_; + std::deque free_tags_; + }; +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/src/locality.cpp b/libs/full/parcelport_openshmem/src/locality.cpp new file mode 100644 index 000000000000..e9e22c16acbe --- /dev/null +++ b/libs/full/parcelport_openshmem/src/locality.cpp @@ -0,0 +1,36 @@ +// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2007-2021 Hartmut Kaiser +// Copyright (c) 2013-2014 Thomas Heller +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include +#include + +namespace hpx::parcelset::policies::openshmem { + + void locality::save(serialization::output_archive& ar) const + { + ar << rank_; + } + + void locality::load(serialization::input_archive& ar) + { + ar >> rank_; + } + + std::ostream& operator<<(std::ostream& os, locality const& loc) noexcept + { + hpx::util::ios_flags_saver ifs(os); + os << loc.rank_; + return os; + } +} // namespace hpx::parcelset::policies::openshmem + +#endif diff --git a/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp b/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp new file mode 100644 index 000000000000..27b2f2de4a3e --- /dev/null +++ b/libs/full/parcelport_openshmem/src/parcelport_openshmem.cpp @@ -0,0 +1,313 @@ +// Copyright (c) 2023 Christopher Taylor +// Copyright (c) 2007-2022 Hartmut Kaiser +// Copyright (c) 2014-2015 Thomas Heller +// Copyright (c) 2020 Google +// +// SPDX-License-Identifier: BSL-1.0 +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +#include + +#if defined(HPX_HAVE_NETWORKING) && defined(HPX_HAVE_PARCELPORT_OPENSHMEM) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace hpx::parcelset { + + namespace policies::openshmem { + class HPX_EXPORT parcelport; + } // namespace policies::openshmem + + template <> + struct connection_handler_traits + { + using connection_type = policies::openshmem::sender_connection; + using send_early_parcel = std::true_type; + using do_background_work = std::true_type; + using send_immediate_parcels = std::false_type; + using is_connectionless = std::false_type; + + static constexpr const char* type() noexcept + { + return "openshmem"; + } + + static constexpr const char* pool_name() noexcept + { + return "parcel-pool-openshmem"; + } + + static constexpr const char* pool_name_postfix() noexcept + { + return "-openshmem"; + } + }; + + namespace policies::openshmem { + + int acquire_tag(sender* s) noexcept + { + return s->acquire_tag(); + } + + void add_connection( + sender* s, std::shared_ptr const& ptr) + { + s->add(ptr); + } + + class HPX_EXPORT parcelport : public parcelport_impl + { + using base_type = parcelport_impl; + + static parcelset::locality here() + { + return parcelset::locality( + locality(util::openshmem_environment::enabled() ? + util::openshmem_environment::rank() : + -1)); + } + + static std::size_t max_connections( + util::runtime_configuration const& ini) + { + return hpx::util::get_entry_as(ini, + "hpx.parcel.openshmem.max_connections", + HPX_PARCEL_MAX_CONNECTIONS); + } + + static std::size_t background_threads( + util::runtime_configuration const& ini) + { + /* + return hpx::util::get_entry_as(ini, + "hpx.parcel.openshmem.background_threads", + HPX_HAVE_PARCELPORT_OPENSHMEM_BACKGROUND_THREADS); + */ + return 1UL; + } + + public: + parcelport(util::runtime_configuration const& ini, + threads::policies::callback_notifier const& notifier) + : base_type(ini, here(), notifier) + , stopped_(false) + , receiver_(*this) + , background_threads_(background_threads(ini)) + { + } + + parcelport(parcelport const&) = delete; + parcelport(parcelport&&) = delete; + parcelport& operator=(parcelport const&) = delete; + parcelport& operator=(parcelport&&) = delete; + + ~parcelport() + { + util::openshmem_environment::finalize(); + } + + // Start the handling of connections. + bool do_run() + { + receiver_.run(); + sender_.run(); + + for (std::size_t i = 0; i != io_service_pool_.size(); ++i) + { + io_service_pool_.get_io_service(int(i)).post( + hpx::bind(&parcelport::io_service_work, this)); + } + return true; + } + + // Stop the handling of connections. + void do_stop() + { + while (do_background_work(0, parcelport_background_mode_all)) + { + if (threads::get_self_ptr()) + hpx::this_thread::suspend( + hpx::threads::thread_schedule_state::pending, + "openshmem::parcelport::do_stop"); + } + + bool expected = false; + if (stopped_.compare_exchange_strong(expected, true)) + { + stopped_ = true; + hpx::util::openshmem_environment::global_barrier(); + } + } + + /// Return the name of this locality + std::string get_locality_name() const override + { + return util::openshmem_environment::get_processor_name(); + } + + std::shared_ptr create_connection( + parcelset::locality const& l, error_code&) + { + int dest_rank = l.get().rank(); + return sender_.create_connection(dest_rank, this); + } + + parcelset::locality agas_locality( + util::runtime_configuration const&) const override + { + return parcelset::locality( + locality(util::openshmem_environment::enabled() ? 0 : -1)); + } + + parcelset::locality create_locality() const override + { + return parcelset::locality(locality()); + } + + bool background_work( + std::size_t num_thread, parcelport_background_mode mode) + { + if (stopped_.load(std::memory_order_acquire) || + num_thread >= background_threads_) + { + return false; + } + + bool has_work = false; + if (mode & parcelport_background_mode_send) + { + has_work = sender_.background_work(); + } + if (mode & parcelport_background_mode_receive) + { + has_work = receiver_.background_work() || has_work; + } + return has_work; + } + + private: + std::atomic stopped_; + + sender sender_; + receiver receiver_; + + void io_service_work() + { + std::size_t k = 0; + + // We only execute work on the IO service while HPX is starting + while (hpx::is_starting()) + { + bool has_work = sender_.background_work(); + has_work = receiver_.background_work() || has_work; + if (has_work) + { + k = 0; + } + else + { + ++k; + util::detail::yield_k(k, + "hpx::parcelset::policies::openshmem::parcelport::" + "io_service_work"); + } + } + } + + std::size_t background_threads_; + + void early_write_handler(std::error_code const& ec, parcel const& p) + { + if (ec) + { + // all errors during early parcel handling are fatal + std::exception_ptr exception = hpx::detail::get_exception( + hpx::exception(ec), "openshmem::early_write_handler", + __FILE__, __LINE__, + "error while handling early parcel: " + ec.message() + + "(" + std::to_string(ec.value()) + ")" + + parcelset::dump_parcel(p)); + + hpx::report_error(exception); + } + } + }; + } // namespace policies::openshmem +} // namespace hpx::parcelset + +namespace hpx::traits { + + // Inject additional configuration data into the factory registry for this + // type. This information ends up in the system wide configuration database + // under the plugin specific section: + // + // [hpx.parcel.openshmem] + // ... + // priority = 1 + // + template <> + struct plugin_config_data + { + static constexpr char const* priority() noexcept + { + return "1"; + } + + static void init( + int* argc, char*** argv, util::command_line_handling& cfg) + { + util::openshmem_environment::init(argc, argv, cfg.rtcfg_); + cfg.num_localities_ = + static_cast(util::openshmem_environment::size()); + cfg.node_ = + static_cast(util::openshmem_environment::rank()); + } + + // by default no additional initialization using the resource + // partitioner is required + static constexpr void init(hpx::resource::partitioner&) noexcept {} + + static void destroy() noexcept + { + util::openshmem_environment::finalize(); + } + + static constexpr char const* call() noexcept + { + return ""; + } + }; +} // namespace hpx::traits + +HPX_REGISTER_PARCELPORT( + hpx::parcelset::policies::openshmem::parcelport, openshmem) + +#endif diff --git a/libs/full/parcelport_openshmem/tests/CMakeLists.txt b/libs/full/parcelport_openshmem/tests/CMakeLists.txt new file mode 100644 index 000000000000..5206a56dbe16 --- /dev/null +++ b/libs/full/parcelport_openshmem/tests/CMakeLists.txt @@ -0,0 +1,42 @@ +# Copyright (c) 2020-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + +include(HPX_Message) + +if(HPX_WITH_TESTS) + if(HPX_WITH_TESTS_UNIT) + add_hpx_pseudo_target(tests.unit.modules.parcelport_openshmem) + add_hpx_pseudo_dependencies( + tests.unit.modules tests.unit.modules.parcelport_openshmem + ) + add_subdirectory(unit) + endif() + + if(HPX_WITH_TESTS_REGRESSIONS) + add_hpx_pseudo_target(tests.regressions.modules.parcelport_openshmem) + add_hpx_pseudo_dependencies( + tests.regressions.modules tests.regressions.modules.parcelport_openshmem + ) + add_subdirectory(regressions) + endif() + + if(HPX_WITH_TESTS_BENCHMARKS) + add_hpx_pseudo_target(tests.performance.modules.parcelport_openshmem) + add_hpx_pseudo_dependencies( + tests.performance.modules tests.performance.modules.parcelport_openshmem + ) + add_subdirectory(performance) + endif() + + if(HPX_WITH_TESTS_HEADERS) + add_hpx_header_tests( + modules.parcelport_openshmem + HEADERS ${parcelport_openshmem_headers} + HEADER_ROOT ${PROJECT_SOURCE_DIR}/include + DEPENDENCIES hpx_parcelport_openshmem + ) + endif() +endif() diff --git a/libs/full/parcelport_openshmem/tests/performance/CMakeLists.txt b/libs/full/parcelport_openshmem/tests/performance/CMakeLists.txt new file mode 100644 index 000000000000..2f7420810a42 --- /dev/null +++ b/libs/full/parcelport_openshmem/tests/performance/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/full/parcelport_openshmem/tests/regressions/CMakeLists.txt b/libs/full/parcelport_openshmem/tests/regressions/CMakeLists.txt new file mode 100644 index 000000000000..2f7420810a42 --- /dev/null +++ b/libs/full/parcelport_openshmem/tests/regressions/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) diff --git a/libs/full/parcelport_openshmem/tests/unit/CMakeLists.txt b/libs/full/parcelport_openshmem/tests/unit/CMakeLists.txt new file mode 100644 index 000000000000..2f7420810a42 --- /dev/null +++ b/libs/full/parcelport_openshmem/tests/unit/CMakeLists.txt @@ -0,0 +1,5 @@ +# Copyright (c) 2020-2021 The STE||AR-Group +# +# SPDX-License-Identifier: BSL-1.0 +# Distributed under the Boost Software License, Version 1.0. (See accompanying +# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) From 901ba47e350257d2190b92f6644e2f0b4357ee5b Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 23 Oct 2023 22:27:05 -0400 Subject: [PATCH 2/6] fixes to cmake script --- cmake/HPX_SetupOpenSHMEM.cmake | 126 +++++++++++++-------------------- 1 file changed, 51 insertions(+), 75 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 9c603343ef33..ec8448d8f4b8 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -26,7 +26,7 @@ macro(hpx_setup_openshmem) pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL osss-ucx) elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") set(OPENSHMEM_PC - "sandia-openshmem"") + "sandia-openshmem") pkg_search_module( OPENSHMEM IMPORTED_TARGET GLOBAL @@ -35,7 +35,19 @@ macro(hpx_setup_openshmem) endif() endif() - if(NOT OPENSHMEM_FOUND) + if((NOT OPENSHMEM_FOUND) AND HPX_WITH_FETCH_OPENSHMEM) + if(NOT CMAKE_C_COMPILER) + message( + FATAL_ERROR + "HPX_WITH_FETCH_OPENSHMEM requires `-DCMAKE_C_COMPILER` to be set; CMAKE_C_COMPILER is currently unset." + ) + endif() + if(NOT CMAKE_CXX_COMPILER) + message( + FATAL_ERROR + "HPX_WITH_FETCH_OPENSHMEM requires `-DCMAKE_CXX_COMPILER` to be set; CMAKE_CXX_COMPILER is currently unset." + ) + endif() find_program(MAKE_EXECUTABLE NAMES gmake make mingw32-make REQUIRED) @@ -53,8 +65,10 @@ macro(hpx_setup_openshmem) if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") - message(STATUS "Fetching - OSSS-UCX-OpenSHMEM") + message( + STATUS + "Fetching OSSS-UCX-OpenSHMEM" + ) fetchcontent_declare( openshmem @@ -62,18 +76,17 @@ macro(hpx_setup_openshmem) URL https://github.com/openshmem-org/osss-ucx/archive/refs/tags/v1.0.2.tar.gz ) - message(STATUS "Building - OSSS-UCX - (OpenSHMEM on UCX) - and - installing - into - ${CMAKE_INSTALL_PREFIX}") + message( + STATUS + "Building OSSS-UCX (OpenSHMEM on UCX) and installing into ${CMAKE_INSTALL_PREFIX}" + ) elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") - message(STATUS "Fetching - Sandia-OpenSHMEM") + message( + STATUS + "Fetching Sandia-OpenSHMEM" + ) fetchcontent_declare( openshmem @@ -81,23 +94,16 @@ macro(hpx_setup_openshmem) URL https://github.com/Sandia-OpenSHMEM/SOS/archive/refs/tags/v1.5.2.tar.gz ) - message(STATUS "Building - and - installing - Sandia - OpenSHMEM - into - ${CMAKE_INSTALL_PREFIX}") + message( + STATUS + "Building and installing Sandia OpenSHMEM into ${CMAKE_INSTALL_PREFIX}" + ) else() - message(FATAL_ERROR "HPX_WITH_PARCELPORT_OPENSHMEM - is - not - set - to - `ucx` - or - `sos`") + message( + FATAL_ERROR + "HPX_WITH_PARCELPORT_OPENSHMEM is not set to `ucx` or `sos`" + ) endif() fetchcontent_getproperties(openshmem) @@ -115,19 +121,7 @@ macro(hpx_setup_openshmem) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} - ./autogen.sh - && - CC=${CMAKE_C_COMPILER} - ./configure - --prefix=${OPENSHMEM_DIR}/install - --enable-shared - ${PMI_AUTOCONF_OPTS} - && - make - && - make - install" + "./autogen.sh && CC=${CMAKE_C_COMPILER} ./configure --prefix=${OPENSHMEM_DIR}/install --enable-shared ${PMI_AUTOCONF_OPTS} && make && make install" WORKING_DIRECTORY ${OPENSHMEM_DIR} RESULT_VARIABLE OPENSHMEM_BUILD_STATUS OUTPUT_FILE ${OPENSHMEM_BUILD_OUTPUT} @@ -135,17 +129,10 @@ macro(hpx_setup_openshmem) ) if(OPENSHMEM_BUILD_STATUS) - message(FATAL_ERROR "OpenSHMEM - build - result - = - ${OPENSHMEM_SRC_BUILD_STATUS} - - - see - ${OPENSHMEM_SRC_BUILD_OUTPUT} - for - more - details") + message( + FATAL_ERROR + "OpenSHMEM build result = ${OPENSHMEM_SRC_BUILD_STATUS} - see ${OPENSHMEM_SRC_BUILD_OUTPUT} for more details" + ) else() find_file(OPENSHMEM_PKGCONFIG_FILE_FOUND @@ -155,22 +142,12 @@ macro(hpx_setup_openshmem) if(NOT OPENSHMEM_PKGCONFIG_FILE_FOUND) message( - FATAL_ERROR - "PKG-CONFIG - ERROR - (${OPENSHMEM_PKGCONFIG_FILE_FOUND}) - -> - CANNOT - FIND - COMPILED - OpenSHMEM: - ${OPENSHMEMT_DIR}/install/lib/pkgconfig" + FATAL_ERROR + "PKG-CONFIG ERROR (${OPENSHMEM_PKGCONFIG_FILE_FOUND}) -> CANNOT FIND COMPILED OpenSHMEM: ${OPENSHMEMT_DIR}/install/lib/pkgconfig" ) endif() - install(CODE "set - (OPENSHMEM_PATH \"${OPENSHMEM_DIR}\") - ") + install(CODE "set(OPENSHMEM_PATH \"${OPENSHMEM_DIR}\")") install( CODE [[ @@ -181,9 +158,10 @@ macro(hpx_setup_openshmem) ) if(NOT OPENSHMEM_PKGCONFIG_FILE_CONTENT) - message(FATAL_ERROR "ERROR - INSTALLING - OPENSHMEM") + message( + FATAL_ERROR + "ERROR INSTALLING OPENSHMEM" + ) endif() string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" @@ -200,9 +178,10 @@ macro(hpx_setup_openshmem) file(GLOB_RECURSE OPENSHMEM_FILES ${OPENSHMEM_PATH}/install/*) if(NOT OPENSHMEM_FILES) - message(STATUS "ERROR - INSTALLING - OPENSHMEM") + message( + STATUS + "ERROR INSTALLING OPENSHMEM" + ) endif() foreach(OPENSHMEM_FILE ${OPENSHMEM_FILES}) @@ -992,11 +971,8 @@ macro(hpx_setup_openshmem) PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS "${OPENSHMEM_LDFLAGS}" ) set_target_properties( - PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES - "${OPENSHMEM_LIBRARY_DIRS} - " + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES "${OPENSHMEM_LIBRARY_DIRS}" ) endif() - endif() endmacro() From 68d12fcb09c6652eed755f05bfce21dc1baa79fb Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Mon, 23 Oct 2023 22:33:00 -0400 Subject: [PATCH 3/6] cmake-format applied --- cmake/HPX_SetupOpenSHMEM.cmake | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index ec8448d8f4b8..6c10f6eace72 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -232,15 +232,9 @@ macro(hpx_setup_openshmem) endif() if(NOT OPENSHMEM_FOUND) - message(FATAL_ERROR "OpenSHMEM - downloaded, - compiled, - but - cannot - be - found - in - ${CMAKE_INSTALL_PREFIX}") + message( + FATAL_ERROR "OpenSHMEM downloaded, compiled, but cannot be found in ${CMAKE_INSTALL_PREFIX}" + ) endif() endif() From 1811c270fab2b77f4dfac720e94362ee3c65e4af Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Wed, 25 Oct 2023 10:37:00 -0400 Subject: [PATCH 4/6] fixed misquote --- cmake/HPX_SetupOpenSHMEM.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 6c10f6eace72..54133f93154a 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -53,9 +53,9 @@ macro(hpx_setup_openshmem) include(FindOpenShmemPmi) - set(PMI_AUTOCONF_OPTS) + set(PMI_AUTOCONF_OPTS "") if(NOT PMI_LIBRARY OR NOT PMI_FOUND) - set(PMI_AUTOCONF_OPTS --enable-pmi-simple) + set(PMI_AUTOCONF_OPTS "--enable-pmi-simple") else() set(PMI_AUTOCONF_OPTS "--with-pmi=${PMI_INCLUDE_DIR} --with-pmi-libdir=${PMI_LIBRARY}") @@ -143,7 +143,7 @@ macro(hpx_setup_openshmem) if(NOT OPENSHMEM_PKGCONFIG_FILE_FOUND) message( FATAL_ERROR - "PKG-CONFIG ERROR (${OPENSHMEM_PKGCONFIG_FILE_FOUND}) -> CANNOT FIND COMPILED OpenSHMEM: ${OPENSHMEMT_DIR}/install/lib/pkgconfig" + "PKG-CONFIG ERROR (${OPENSHMEM_PKGCONFIG_FILE_FOUND}) -> CANNOT FIND COMPILED OpenSHMEM: ${OPENSHMEM_DIR}/install/lib/pkgconfig" ) endif() From 0a43bc0d30f31d1aa01d26ab0ed49b0542ce967a Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Wed, 25 Oct 2023 15:03:10 -0400 Subject: [PATCH 5/6] cmake formatting reapplied --- CMakeLists.txt | 3 +- cmake/HPX_SetupOpenSHMEM.cmake | 1186 ++++++++++++++++---------------- 2 files changed, 596 insertions(+), 593 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8c27847f7621..fb2fe8b4b818 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1196,8 +1196,7 @@ if(HPX_WITH_NETWORKING) OFF CATEGORY "Parcelport" ) hpx_option( - HPX_WITH_FETCH_OPENSHMEM - BOOL + HPX_WITH_FETCH_OPENSHMEM BOOL "Download an OpenSHMEM if one is not available" OFF CATEGORY "Build Targets" ADVANCED diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 54133f93154a..5046968a45df 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -25,13 +25,9 @@ macro(hpx_setup_openshmem) pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL osss-ucx) elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") - set(OPENSHMEM_PC - "sandia-openshmem") + set(OPENSHMEM_PC "sandia-openshmem") - pkg_search_module( - OPENSHMEM IMPORTED_TARGET GLOBAL - sandia-openshmem - ) + pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL sandia-openshmem) endif() endif() @@ -58,35 +54,30 @@ macro(hpx_setup_openshmem) set(PMI_AUTOCONF_OPTS "--enable-pmi-simple") else() set(PMI_AUTOCONF_OPTS "--with-pmi=${PMI_INCLUDE_DIR} - --with-pmi-libdir=${PMI_LIBRARY}") + --with-pmi-libdir=${PMI_LIBRARY}" + ) endif() include(FetchContent) if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") - message( - STATUS - "Fetching OSSS-UCX-OpenSHMEM" - ) + message(STATUS "Fetching OSSS-UCX-OpenSHMEM") fetchcontent_declare( openshmem DOWNLOAD_EXTRACT_TIMESTAMP TRUE - URL https://github.com/openshmem-org/osss-ucx/archive/refs/tags/v1.0.2.tar.gz + URL https://github.com/openshmem-org/osss-ucx/archive/refs/tags/v1.0.2.tar.gz ) message( - STATUS - "Building OSSS-UCX (OpenSHMEM on UCX) and installing into ${CMAKE_INSTALL_PREFIX}" + STATUS + "Building OSSS-UCX (OpenSHMEM on UCX) and installing into ${CMAKE_INSTALL_PREFIX}" ) elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") - message( - STATUS - "Fetching Sandia-OpenSHMEM" - ) + message(STATUS "Fetching Sandia-OpenSHMEM") fetchcontent_declare( openshmem @@ -95,15 +86,14 @@ macro(hpx_setup_openshmem) ) message( - STATUS - "Building and installing Sandia OpenSHMEM into ${CMAKE_INSTALL_PREFIX}" + STATUS + "Building and installing Sandia OpenSHMEM into ${CMAKE_INSTALL_PREFIX}" ) else() - message( - FATAL_ERROR - "HPX_WITH_PARCELPORT_OPENSHMEM is not set to `ucx` or `sos`" - ) + message( + FATAL_ERROR "HPX_WITH_PARCELPORT_OPENSHMEM is not set to `ucx` or `sos`" + ) endif() fetchcontent_getproperties(openshmem) @@ -130,20 +120,19 @@ macro(hpx_setup_openshmem) if(OPENSHMEM_BUILD_STATUS) message( - FATAL_ERROR - "OpenSHMEM build result = ${OPENSHMEM_SRC_BUILD_STATUS} - see ${OPENSHMEM_SRC_BUILD_OUTPUT} for more details" + FATAL_ERROR + "OpenSHMEM build result = ${OPENSHMEM_SRC_BUILD_STATUS} - see ${OPENSHMEM_SRC_BUILD_OUTPUT} for more details" ) else() - find_file(OPENSHMEM_PKGCONFIG_FILE_FOUND - ${OPENSHMEM_PC} - ${OPENSHMEM_DIR}/install/lib/pkgconfig + find_file(OPENSHMEM_PKGCONFIG_FILE_FOUND ${OPENSHMEM_PC} + ${OPENSHMEM_DIR}/install/lib/pkgconfig ) if(NOT OPENSHMEM_PKGCONFIG_FILE_FOUND) message( - FATAL_ERROR - "PKG-CONFIG ERROR (${OPENSHMEM_PKGCONFIG_FILE_FOUND}) -> CANNOT FIND COMPILED OpenSHMEM: ${OPENSHMEM_DIR}/install/lib/pkgconfig" + FATAL_ERROR + "PKG-CONFIG ERROR (${OPENSHMEM_PKGCONFIG_FILE_FOUND}) -> CANNOT FIND COMPILED OpenSHMEM: ${OPENSHMEM_DIR}/install/lib/pkgconfig" ) endif() @@ -220,118 +209,115 @@ macro(hpx_setup_openshmem) set(ENV{PKG_CONFIG_PATH} "${OPENSHMEM_DIR}/install/lib/pkgconfig") if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") - pkg_search_module( - OPENSHMEM IMPORTED_TARGET GLOBAL - osss-ucx - ) + pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL osss-ucx) elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") - pkg_search_module( - OPENSHMEM IMPORTED_TARGET GLOBAL - sandia-openshmem - ) + pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL sandia-openshmem) endif() if(NOT OPENSHMEM_FOUND) message( - FATAL_ERROR "OpenSHMEM downloaded, compiled, but cannot be found in ${CMAKE_INSTALL_PREFIX}" + FATAL_ERROR + "OpenSHMEM downloaded, compiled, but cannot be found in ${CMAKE_INSTALL_PREFIX}" ) endif() endif() - if(OPENSHMEM_CFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") - - foreach(X IN ITEMS ${OPENSHMEM_CFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" + if(OPENSHMEM_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}" ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} - ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH OPENSHMEM_CFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT OPENSHMEM_CFLAGS NEWPARAM) - endforeach() + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND OPENSHMEM_CFLAGS "${X}") - endforeach() - endif() + list(LENGTH OPENSHMEM_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_CFLAGS NEWPARAM) + endforeach() - if(OPENSHMEM_CFLAGS_OTHER) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_CFLAGS "${X}") + endforeach() + endif() - foreach(X IN ITEMS ${OPENSHMEM_CFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" + if(OPENSHMEM_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}" ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} - ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() - list(LENGTH OPENSHMEM_CFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT OPENSHMEM_CFLAGS_OTHER NEWPARAM) - endforeach() + list(LENGTH OPENSHMEM_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_CFLAGS_OTHER NEWPARAM) + endforeach() - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND OPENSHMEM_CFLAGS_OTHER "${X}") - endforeach() - endif() + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_CFLAGS_OTHER "${X}") + endforeach() + endif() - if(OPENSHMEM_LDFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(DIRIDX 0) - set(SKIP 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${OPENSHMEM_LDFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lsma" IDX) - string(FIND "${X}" "-l" LIDX) - string(FIND "${X}" "-L" DIRIDX) - string(FIND "${X}" "-Wl" SKIP) - - if("${SKIP}" EQUAL "-1") + if(OPENSHMEM_LDFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(DIRIDX 0) + set(SKIP 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lsma" IDX) + string(FIND "${X}" "-l" LIDX) + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") if(NOT "${PARAM_FOUND}" EQUAL "-1") set(IS_PARAM "1") set(NEWPARAM "SHELL:${X}") @@ -345,7 +331,8 @@ macro(hpx_setup_openshmem) set(IS_PARAM "0") elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") list(APPEND FLAG_LIST "${NEWPARAM} - ${X}") + ${X}" + ) set(NEWPARAM "") set(IS_PARAM "0") elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") @@ -361,112 +348,114 @@ macro(hpx_setup_openshmem) string(REPLACE "-L" "" TMPSTR "${X}") list(APPEND DIR_LIST "${TMPSTR}") endif() - endif() - endforeach() + endif() + endforeach() - set(IDX 0) - list(LENGTH OPENSHMEM_LDFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT OPENSHMEM_LDFLAGS NEWPARAM) - endforeach() + set(IDX 0) + list(LENGTH OPENSHMEM_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_LDFLAGS NEWPARAM) + endforeach() - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND OPENSHMEM_LDFLAGS "${X}") - endforeach() + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_LDFLAGS "${X}") + endforeach() + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive - ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " - " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() + + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + " + ) + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" + ) + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() endforeach() - string(APPEND NEWLINK " - -Wl,--no-whole-archive") - string(FIND "SHELL:-Wl,--whole-archive + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive" - "${NEWLINK}" IDX + ) + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") + endif() + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + " ) - if("${IDX}" EQUAL "-1") - list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: - ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " - " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" ) - if("${IDX}" EQUAL "-1") - list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") - endif() + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" "${NEWLINK}" IDX) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") endif() endif() endif() + endif() - if(OPENSHMEM_LDFLAGS_OTHER) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(SKIP 0) - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${OPENSHMEM_LDFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lsma" IDX) - string(FIND "${X}" "-L" DIRIDX) - string(FIND "${X}" "-Wl" SKIP) - - if("${SKIP}" EQUAL "-1") + if(OPENSHMEM_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(SKIP 0) + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lsma" IDX) + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") if(NOT "${PARAM_FOUND}" EQUAL "-1") set(IS_PARAM "1") set(NEWPARAM "SHELL:${X}") @@ -480,7 +469,8 @@ macro(hpx_setup_openshmem) set(IS_PARAM "0") elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") list(APPEND FLAG_LIST "${NEWPARAM} - ${X}") + ${X}" + ) set(NEWPARAM "") set(IS_PARAM "0") elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") @@ -496,187 +486,191 @@ macro(hpx_setup_openshmem) string(REPLACE "-L" "" TMPSTR "${X}") list(APPEND DIR_LIST "${TMPSTR}") endif() - endif() - endforeach() + endif() + endforeach() - set(IDX 0) - list(LENGTH OPENSHMEM_LDFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT OPENSHMEM_LDFLAGS_OTHER NEWPARAM) - endforeach() + set(IDX 0) + list(LENGTH OPENSHMEM_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_LDFLAGS_OTHER NEWPARAM) + endforeach() - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND OPENSHMEM_LDFLAGS_OTHER "${X}") - endforeach() + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_LDFLAGS_OTHER "${X}") + endforeach() + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive - ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " - " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + " + ) + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" + ) + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() endforeach() - string(APPEND NEWLINK " - -Wl,--no-whole-archive") - - string(FIND "SHELL:-Wl,--whole-archive + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive" - "${NEWLINK}" IDX + ) + + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS_OTHER "${NEWLINK}") + endif() + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + " ) - if("${IDX}" EQUAL "-1") - list(APPEND OPENSHMEM_LDFLAGS_OTHER "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: - ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " - " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" ) - if("${IDX}" EQUAL "-1") - list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") - endif() + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" "${NEWLINK}" IDX) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") endif() endif() - endif() - if(OPENSHMEM_STATIC_CFLAGS) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") + endif() - foreach(X IN ITEMS ${OPENSHMEM_STATIC_CFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" + if(OPENSHMEM_STATIC_CFLAGS) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_STATIC_CFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}" ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} - ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() - - list(LENGTH OPENSHMEM_STATIC_CFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT OPENSHMEM_STATIC_CFLAGS NEWPARAM) - endforeach() + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND OPENSHMEM_STATIC_CFLAGS "${X}") - endforeach() - endif() + list(LENGTH OPENSHMEM_STATIC_CFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_STATIC_CFLAGS NEWPARAM) + endforeach() - if(OPENSHMEM_STATIC_CFLAGS_OTHER) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(IDX 0) - set(FLAG_LIST "") + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_STATIC_CFLAGS "${X}") + endforeach() + endif() - foreach(X IN ITEMS ${OPENSHMEM_STATIC_CFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - if(NOT "${PARAM_FOUND}" EQUAL "-1") - set(IS_PARAM "1") - set(NEWPARAM "SHELL:${X}") - endif() - if("${PARAM_FOUND}" EQUAL "-1" - AND "${IS_PARAM}" EQUAL "0" - OR "${IS_PARAM}" EQUAL "-1" + if(OPENSHMEM_STATIC_CFLAGS_OTHER) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(IDX 0) + set(FLAG_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_STATIC_CFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + if(NOT "${PARAM_FOUND}" EQUAL "-1") + set(IS_PARAM "1") + set(NEWPARAM "SHELL:${X}") + endif() + if("${PARAM_FOUND}" EQUAL "-1" + AND "${IS_PARAM}" EQUAL "0" + OR "${IS_PARAM}" EQUAL "-1" + ) + list(APPEND FLAG_LIST "${X}") + set(IS_PARAM "0") + elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") + list(APPEND FLAG_LIST "${NEWPARAM} + ${X}" ) - list(APPEND FLAG_LIST "${X}") - set(IS_PARAM "0") - elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") - list(APPEND FLAG_LIST "${NEWPARAM} - ${X}") - set(NEWPARAM "") - set(IS_PARAM "0") - endif() - endforeach() + set(NEWPARAM "") + set(IS_PARAM "0") + endif() + endforeach() - list(LENGTH OPENSHMEM_STATIC_CFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT OPENSHMEM_STATIC_CFLAGS_OTHER NEWPARAM) - endforeach() + list(LENGTH OPENSHMEM_STATIC_CFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_STATIC_CFLAGS_OTHER NEWPARAM) + endforeach() - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND OPENSHMEM_STATIC_CFLAGS_OTHER "${X}") - endforeach() - endif() + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_STATIC_CFLAGS_OTHER "${X}") + endforeach() + endif() - if(OPENSHMEM_STATIC_LDFLAGS) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(SKIP 0) - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${OPENSHMEM_STATIC_LDFLAGS}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lsma" IDX) - string(FIND "${X}" "-L" DIRIDX) - string(FIND "${X}" "-Wl" SKIP) - - if("${SKIP}" EQUAL "-1") + if(OPENSHMEM_STATIC_LDFLAGS) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(SKIP 0) + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_STATIC_LDFLAGS}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lsma" IDX) + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") if(NOT "${PARAM_FOUND}" EQUAL "-1") set(IS_PARAM "1") set(NEWPARAM "SHELL:${X}") @@ -690,7 +684,8 @@ macro(hpx_setup_openshmem) set(IS_PARAM "0") elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") list(APPEND FLAG_LIST "${NEWPARAM} - ${X}") + ${X}" + ) set(NEWPARAM "") set(IS_PARAM "0") elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") @@ -706,113 +701,115 @@ macro(hpx_setup_openshmem) string(REPLACE "-L" "" TMPSTR "${X}") list(APPEND DIR_LIST "${TMPSTR}") endif() - endif() - endforeach() + endif() + endforeach() - set(IDX 0) - list(LENGTH OPENSHMEM_STATIC_LDFLAGS IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT OPENSHMEM_STATIC_LDFLAGS NEWPARAM) - endforeach() + set(IDX 0) + list(LENGTH OPENSHMEM_STATIC_LDFLAGS IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_STATIC_LDFLAGS NEWPARAM) + endforeach() - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND OPENSHMEM_STATIC_LDFLAGS "${X}") - endforeach() + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_STATIC_LDFLAGS "${X}") + endforeach() + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive - ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " - " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + " + ) + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" + ) + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() endforeach() - string(APPEND NEWLINK " - -Wl,--no-whole-archive") - - string(FIND "SHELL:-Wl,--whole-archive + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive" - "${NEWLINK}" IDX + ) + + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_STATIC_LDFLAGS "${NEWLINK}") + endif() + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + " ) - if("${IDX}" EQUAL "-1") - list(APPEND OPENSHMEM_STATIC_LDFLAGS "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: - ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " - " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" ) - if("${IDX}" EQUAL "-1") - list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") - endif() + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" "${NEWLINK}" IDX) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") endif() endif() endif() + endif() - if(OPENSHMEM_STATIC_LDFLAGS_OTHER) - unset(FOUND_LIB) - set(IS_PARAM "0") - set(PARAM_FOUND "0") - set(NEWPARAM "") - set(SKIP 0) - set(IDX 0) - set(DIRIDX 0) - set(FLAG_LIST "") - set(DIR_LIST "") - set(LIB_LIST "") - - foreach(X IN ITEMS ${OPENSHMEM_STATIC_LDFLAGS_OTHER}) - string(FIND "${X}" "--param" PARAM_FOUND) - string(FIND "${X}" "-lsma" IDX) - string(FIND "${X}" "-L" DIRIDX) - string(FIND "${X}" "-Wl" SKIP) - - if("${SKIP}" EQUAL "-1") + if(OPENSHMEM_STATIC_LDFLAGS_OTHER) + unset(FOUND_LIB) + set(IS_PARAM "0") + set(PARAM_FOUND "0") + set(NEWPARAM "") + set(SKIP 0) + set(IDX 0) + set(DIRIDX 0) + set(FLAG_LIST "") + set(DIR_LIST "") + set(LIB_LIST "") + + foreach(X IN ITEMS ${OPENSHMEM_STATIC_LDFLAGS_OTHER}) + string(FIND "${X}" "--param" PARAM_FOUND) + string(FIND "${X}" "-lsma" IDX) + string(FIND "${X}" "-L" DIRIDX) + string(FIND "${X}" "-Wl" SKIP) + + if("${SKIP}" EQUAL "-1") if(NOT "${PARAM_FOUND}" EQUAL "-1") set(IS_PARAM "1") set(NEWPARAM "SHELL:${X}") @@ -826,7 +823,8 @@ macro(hpx_setup_openshmem) set(IS_PARAM "0") elseif("${PARAM_FOUND}" EQUAL "-1" AND "${IS_PARAM}" EQUAL "1") list(APPEND FLAG_LIST "${NEWPARAM} - ${X}") + ${X}" + ) set(NEWPARAM "") set(IS_PARAM "0") elseif(NOT "${IDX}" EQUAL "-1" AND NOT "${LIDX}" EQUAL "-1") @@ -842,131 +840,137 @@ macro(hpx_setup_openshmem) string(REPLACE "-L" "" TMPSTR "${X}") list(APPEND DIR_LIST "${TMPSTR}") endif() - endif() - endforeach() + endif() + endforeach() - set(IDX 0) - list(LENGTH OPENSHMEM_STATIC_LDFLAGS_OTHER IDX) - foreach(X RANGE ${IDX}) - list(POP_FRONT OPENSHMEM_STATIC_LDFLAGS_OTHER NEWPARAM) - endforeach() + set(IDX 0) + list(LENGTH OPENSHMEM_STATIC_LDFLAGS_OTHER IDX) + foreach(X RANGE ${IDX}) + list(POP_FRONT OPENSHMEM_STATIC_LDFLAGS_OTHER NEWPARAM) + endforeach() - foreach(X IN ITEMS ${FLAG_LIST}) - list(APPEND OPENSHMEM_STATIC_LDFLAGS_OTHER "${X}") - endforeach() + foreach(X IN ITEMS ${FLAG_LIST}) + list(APPEND OPENSHMEM_STATIC_LDFLAGS_OTHER "${X}") + endforeach() + set(IDX 0) + list(LENGTH LIB_LIST IDX) + if(NOT "${IDX}" EQUAL "0") set(IDX 0) - list(LENGTH LIB_LIST IDX) - if(NOT "${IDX}" EQUAL "0") - set(IDX 0) - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(NEWLINK "SHELL:-Wl,--whole-archive - ") - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " - " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - - message(STATUS "${FOUND_LIB} - ${X}") - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(NEWLINK "SHELL:-Wl,--whole-archive + " + ) + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" + ) + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + + message(STATUS "${FOUND_LIB} + ${X}" + ) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() endforeach() - string(APPEND NEWLINK " - -Wl,--no-whole-archive") - string(FIND "SHELL:-Wl,--whole-archive + endforeach() + string(APPEND NEWLINK " -Wl,--no-whole-archive" - "${NEWLINK}" IDX + ) + string(FIND "SHELL:-Wl,--whole-archive + -Wl,--no-whole-archive" "${NEWLINK}" IDX + ) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_STATIC_LDFLAGS_OTHER "${NEWLINK}") + endif() + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + if(APPLE) + set(NEWLINK "SHELL:-Wl,-force_load,") + else() + set(NEWLINK "SHELL: + " ) - if("${IDX}" EQUAL "-1") - list(APPEND OPENSHMEM_STATIC_LDFLAGS_OTHER "${NEWLINK}") - endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - if(APPLE) - set(NEWLINK "SHELL:-Wl,-force_load,") - else() - set(NEWLINK "SHELL: - ") - endif() - foreach(X IN ITEMS ${LIB_LIST}) - set(DIRSTR "") - string(REPLACE ";" " - " DIRSTR "${DIR_LIST}") - foreach(Y IN ITEMS ${DIR_LIST}) - find_library( - FOUND_LIB - NAMES ${X} "lib${X}" "lib${X}.a" - PATHS ${Y} - HINTS ${Y} NO_CACHE - NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH - ) - - list(LENGTH FOUND_LIB IDX) - if(NOT "${IDX}" EQUAL "0") - string(APPEND NEWLINK "${FOUND_LIB}") - set(FOUND_LIB "") - endif() - endforeach() - endforeach() - string(FIND "SHELL:" - "${NEWLINK}" IDX + endif() + foreach(X IN ITEMS ${LIB_LIST}) + set(DIRSTR "") + string(REPLACE ";" " + " DIRSTR "${DIR_LIST}" ) - if("${IDX}" EQUAL "-1") - list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") - endif() + foreach(Y IN ITEMS ${DIR_LIST}) + find_library( + FOUND_LIB + NAMES ${X} "lib${X}" "lib${X}.a" + PATHS ${Y} + HINTS ${Y} NO_CACHE + NO_CMAKE_FIND_ROOT_PATH NO_DEFAULT_PATH + ) + + list(LENGTH FOUND_LIB IDX) + if(NOT "${IDX}" EQUAL "0") + string(APPEND NEWLINK "${FOUND_LIB}") + set(FOUND_LIB "") + endif() + endforeach() + endforeach() + string(FIND "SHELL:" "${NEWLINK}" IDX) + if("${IDX}" EQUAL "-1") + list(APPEND OPENSHMEM_LDFLAGS "${NEWLINK}") endif() endif() endif() + endif() - if(OPENSHMEM_DIR) - list(TRANSFORM OPENSHMEM_CFLAGS - REPLACE "${OPENSHMEM_DIR}/install" - "$" - ) - list(TRANSFORM OPENSHMEM_LDFLAGS - REPLACE "${OPENSHMEM_DIR}/install" - "$" - ) - list(TRANSFORM OPENSHMEM_LIBRARY_DIRS - REPLACE "${OPENSHMEM_DIR}/install" - "$" - ) + if(OPENSHMEM_DIR) + list(TRANSFORM OPENSHMEM_CFLAGS + REPLACE "${OPENSHMEM_DIR}/install" + "$" + ) + list(TRANSFORM OPENSHMEM_LDFLAGS + REPLACE "${OPENSHMEM_DIR}/install" + "$" + ) + list(TRANSFORM OPENSHMEM_LIBRARY_DIRS + REPLACE "${OPENSHMEM_DIR}/install" + "$" + ) - set_target_properties( - PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS - "${OPENSHMEM_CFLAGS}" - ) - set_target_properties( - PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS - "${OPENSHMEM_LDFLAGS}" - ) - set_target_properties( - PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES - "${OPENSHMEM_LIBRARY_DIRS}" - ) - else() - set_target_properties( - PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS "${OPENSHMEM_CFLAGS}" - ) - set_target_properties( - PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS "${OPENSHMEM_LDFLAGS}" - ) - set_target_properties( - PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES "${OPENSHMEM_LIBRARY_DIRS}" - ) - endif() + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS + "${OPENSHMEM_CFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS + "${OPENSHMEM_LDFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES + "${OPENSHMEM_LIBRARY_DIRS}" + ) + else() + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_COMPILE_OPTIONS + "${OPENSHMEM_CFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_OPTIONS + "${OPENSHMEM_LDFLAGS}" + ) + set_target_properties( + PkgConfig::OPENSHMEM PROPERTIES INTERFACE_LINK_DIRECTORIES + "${OPENSHMEM_LIBRARY_DIRS}" + ) + endif() endmacro() From 2c5c8abe982062403ed07ab57e984bcec871eaf2 Mon Sep 17 00:00:00 2001 From: ct-clmsn Date: Fri, 27 Oct 2023 10:30:52 -0400 Subject: [PATCH 6/6] fixed bugs in cmake script --- cmake/HPX_SetupOpenSHMEM.cmake | 108 ++++++++++++++------------------- 1 file changed, 47 insertions(+), 61 deletions(-) diff --git a/cmake/HPX_SetupOpenSHMEM.cmake b/cmake/HPX_SetupOpenSHMEM.cmake index 5046968a45df..fba9048b521a 100644 --- a/cmake/HPX_SetupOpenSHMEM.cmake +++ b/cmake/HPX_SetupOpenSHMEM.cmake @@ -101,17 +101,16 @@ macro(hpx_setup_openshmem) fetchcontent_populate(openshmem) endif() - set(CMAKE_PREFIX_PATH "${CMAKE_INSTALL_PREFIX}/lib/pkgconfig") - set(ENV{PKG_CONFIG_PATH} "${CMAKE_INSTALL_PREFIX}/lib/pkgconfig") - set(OPENSHMEM_DIR "${openshmem_SOURCE_DIR}") set(OPENSHMEM_BUILD_OUTPUT "${OPENSHMEM_DIR}/build.log") set(OPENSHMEM_ERROR_FILE "${OPENSHMEM_DIR}/error.log") + set(ENV{PKG_CONFIG_PATH} "${OPENSHMEM_DIR}/install/lib/pkgconfig") + execute_process( COMMAND bash -c - "./autogen.sh && CC=${CMAKE_C_COMPILER} ./configure --prefix=${OPENSHMEM_DIR}/install --enable-shared ${PMI_AUTOCONF_OPTS} && make && make install" + "./autogen.sh && CC=${CMAKE_C_COMPILER} ./configure --prefix=${OPENSHMEM_DIR}/install --enable-shared --disable-fortran ${PMI_AUTOCONF_OPTS} && make && make install" WORKING_DIRECTORY ${OPENSHMEM_DIR} RESULT_VARIABLE OPENSHMEM_BUILD_STATUS OUTPUT_FILE ${OPENSHMEM_BUILD_OUTPUT} @@ -125,94 +124,80 @@ macro(hpx_setup_openshmem) ) else() - find_file(OPENSHMEM_PKGCONFIG_FILE_FOUND ${OPENSHMEM_PC} + find_file(OPENSHMEM_PKGCONFIG_FILE_FOUND + ${OPENSHMEM_PC}.pc ${OPENSHMEM_DIR}/install/lib/pkgconfig ) if(NOT OPENSHMEM_PKGCONFIG_FILE_FOUND) message( FATAL_ERROR - "PKG-CONFIG ERROR (${OPENSHMEM_PKGCONFIG_FILE_FOUND}) -> CANNOT FIND COMPILED OpenSHMEM: ${OPENSHMEM_DIR}/install/lib/pkgconfig" + "PKG-CONFIG ERROR (${OPENSHMEM_PKGCONFIG_FILE_FOUND}; ${OPENSHMEM_PC}) -> CANNOT FIND COMPILED OpenSHMEM: ${OPENSHMEM_DIR}/install/lib/pkgconfig" ) endif() + install(CODE "set(OPENSHMEM_PKGCONF \"${OPENSHMEM_PC}\")") install(CODE "set(OPENSHMEM_PATH \"${OPENSHMEM_DIR}\")") install( CODE [[ - file( - READ - ${OPENSHMEM_PATH}/install/lib/pkgconfig/${OPENSHMEM_PC} - OPENSHMEM_PKGCONFIG_FILE_CONTENT - ) + file(READ + ${OPENSHMEM_PATH}/install/lib/pkgconfig/${OPENSHMEM_PKGCONF}.pc + OPENSHMEM_PKGCONFIG_FILE_CONTENT) - if(NOT OPENSHMEM_PKGCONFIG_FILE_CONTENT) - message( - FATAL_ERROR - "ERROR INSTALLING OPENSHMEM" - ) - endif() + if(NOT OPENSHMEM_PKGCONFIG_FILE_CONTENT) + message(FATAL_ERROR "ERROR INSTALLING OPENSHMEM") + endif() - string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" - OPENSHMEM_PKGCONFIG_FILE_CONTENT - ${OPENSHMEM_PKGCONFIG_FILE_CONTENT} - ) + string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" + OPENSHMEM_PKGCONFIG_FILE_CONTENT + ${OPENSHMEM_PKGCONFIG_FILE_CONTENT}) - file( - WRITE - ${OPENSHMEM_PATH}/install/lib/pkgconfig/${OPENSHMEM_PC} - ${OPENSHMEM_PKGCONFIG_FILE_CONTENT} - ) + file(WRITE + ${OPENSHMEM_PATH}/install/lib/pkgconfig/${OPENSHMEM_PKGCONF}.pc + ${OPENSHMEM_PKGCONFIG_FILE_CONTENT} + ) - file(GLOB_RECURSE OPENSHMEM_FILES ${OPENSHMEM_PATH}/install/*) + file(GLOB_RECURSE OPENSHMEM_FILES ${OPENSHMEM_PATH}/install/*) - if(NOT OPENSHMEM_FILES) - message( - STATUS - "ERROR INSTALLING OPENSHMEM" - ) - endif() + if(NOT OPENSHMEM_FILES) + message(STATUS "ERROR INSTALLING OPENSHMEM") + endif() - foreach(OPENSHMEM_FILE ${OPENSHMEM_FILES}) - set(OPENSHMEM_FILE_CACHED "${OPENSHMEM_FILE}") + foreach(OPENSHMEM_FILE ${OPENSHMEM_FILES}) + set(OPENSHMEM_FILE_CACHED "${OPENSHMEM_FILE}") - string(REGEX MATCH " - (^ \/.*\/) - " OPENSHMEM_FILE_PATH ${OPENSHMEM_FILE}) + string(REGEX MATCH "(^\/.*\/)" OPENSHMEM_FILE_PATH ${OPENSHMEM_FILE}) - string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" - OPENSHMEM_FILE ${OPENSHMEM_FILE} - ) + string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" + OPENSHMEM_FILE ${OPENSHMEM_FILE} + ) - string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" - OPENSHMEM_FILE_PATH ${OPENSHMEM_FILE_PATH} - ) + string(REPLACE "${OPENSHMEM_PATH}/install" "${CMAKE_INSTALL_PREFIX}" + OPENSHMEM_FILE_PATH ${OPENSHMEM_FILE_PATH} + ) - file(MAKE_DIRECTORY ${OPENSHMEM_FILE_PATH}) + file(MAKE_DIRECTORY ${OPENSHMEM_FILE_PATH}) - string(LENGTH ${OPENSHMEM_FILE_PATH} OPENSHMEM_FILE_PATH_SIZE) - math(EXPR OPENSHMEM_FILE_PATH_SIZE "${OPENSHMEM_FILE_PATH_SIZE}-1") + string(LENGTH ${OPENSHMEM_FILE_PATH} OPENSHMEM_FILE_PATH_SIZE) + math(EXPR OPENSHMEM_FILE_PATH_SIZE "${OPENSHMEM_FILE_PATH_SIZE}-1") - string(SUBSTRING ${OPENSHMEM_FILE_PATH} 0 ${OPENSHMEM_FILE_PATH_SIZE} - OPENSHMEM_FILE_PATH - ) + string(SUBSTRING ${OPENSHMEM_FILE_PATH} 0 ${OPENSHMEM_FILE_PATH_SIZE} + OPENSHMEM_FILE_PATH) - file(COPY ${OPENSHMEM_FILE_CACHED} DESTINATION ${OPENSHMEM_FILE_PATH}) - endforeach() - ]] + file(COPY ${OPENSHMEM_FILE_CACHED} DESTINATION ${OPENSHMEM_FILE_PATH}) + endforeach() + ]] ) - - # install(FILES ${OPENSHMEM_FILES} DESTINATION ${CMAKE_INSTALL_PREFIX}) endif() set(CMAKE_PREFIX_PATH "${OPENSHMEM_DIR}/install/lib/pkgconfig") set(ENV{PKG_CONFIG_PATH} "${OPENSHMEM_DIR}/install/lib/pkgconfig") - if("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "ucx") - pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL osss-ucx) - elseif("${HPX_WITH_PARCELPORT_OPENSHMEM_CONDUIT}" STREQUAL "sos") - pkg_search_module(OPENSHMEM IMPORTED_TARGET GLOBAL sandia-openshmem) - endif() + pkg_search_module( + OPENSHMEM REQUIRED IMPORTED_TARGET GLOBAL + ${OPENSHMEM_PC} + ) if(NOT OPENSHMEM_FOUND) message( @@ -220,7 +205,8 @@ macro(hpx_setup_openshmem) "OpenSHMEM downloaded, compiled, but cannot be found in ${CMAKE_INSTALL_PREFIX}" ) endif() - + elseif((NOT OPENSHMEM_FOUND) AND (NOT HPX_WITH_FETCH_OPENSHMEM)) + message(FATAL_ERROR "OpenSHMEM not found and HPX_WITH_FETCH_OPENSHMEM not set!") endif() if(OPENSHMEM_CFLAGS)