diff --git a/.jenkins/lsu/env-clang-12.sh b/.jenkins/lsu/env-clang-12.sh index 15e920431f54..c9b5e4a5ee10 100644 --- a/.jenkins/lsu/env-clang-12.sh +++ b/.jenkins/lsu/env-clang-12.sh @@ -25,11 +25,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" # The pwrapi library still needs to be set up properly on rostam diff --git a/.jenkins/lsu/env-clang-13.sh b/.jenkins/lsu/env-clang-13.sh index 4582dd0dba9f..76df09cfd93b 100644 --- a/.jenkins/lsu/env-clang-13.sh +++ b/.jenkins/lsu/env-clang-13.sh @@ -25,11 +25,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_TESTS_COMMAND_LINE=--hpx:queuing=local-workrequesting-fifo" diff --git a/.jenkins/lsu/env-clang-14.sh b/.jenkins/lsu/env-clang-14.sh index 5a52f3c2dca5..144c17e7820d 100644 --- a/.jenkins/lsu/env-clang-14.sh +++ b/.jenkins/lsu/env-clang-14.sh @@ -25,11 +25,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-clang-15.sh b/.jenkins/lsu/env-clang-15.sh index b92c378bd2ce..1df2a9fd23b3 100644 --- a/.jenkins/lsu/env-clang-15.sh +++ b/.jenkins/lsu/env-clang-15.sh @@ -25,11 +25,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-clang-16.sh b/.jenkins/lsu/env-clang-16.sh index c1cf84333a6d..393e42c03e54 100644 --- a/.jenkins/lsu/env-clang-16.sh +++ b/.jenkins/lsu/env-clang-16.sh @@ -25,11 +25,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-clang-17.sh b/.jenkins/lsu/env-clang-17.sh index fc0a9d8eee59..1b6ac8d26294 100644 --- a/.jenkins/lsu/env-clang-17.sh +++ b/.jenkins/lsu/env-clang-17.sh @@ -25,11 +25,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=clang" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" configure_extra_options+=" -DHPX_WITH_LOGGING=OFF" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" diff --git a/.jenkins/lsu/env-gcc-10.sh b/.jenkins/lsu/env-gcc-10.sh index 3953fae9420a..9c4064825a0a 100644 --- a/.jenkins/lsu/env-gcc-10.sh +++ b/.jenkins/lsu/env-gcc-10.sh @@ -25,11 +25,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" # The pwrapi library still needs to be set up properly on rostam # configure_extra_options+=" -DHPX_WITH_POWER_COUNTER=ON" diff --git a/.jenkins/lsu/env-gcc-12.sh b/.jenkins/lsu/env-gcc-12.sh index 80b5c4a9df10..48a6887c1ba6 100644 --- a/.jenkins/lsu/env-gcc-12.sh +++ b/.jenkins/lsu/env-gcc-12.sh @@ -24,11 +24,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" diff --git a/.jenkins/lsu/env-gcc-13.sh b/.jenkins/lsu/env-gcc-13.sh index 032c33b297a7..4a10458aa535 100644 --- a/.jenkins/lsu/env-gcc-13.sh +++ b/.jenkins/lsu/env-gcc-13.sh @@ -24,11 +24,8 @@ configure_extra_options+=" -DHPX_WITH_COMPILER_WARNINGS_AS_ERRORS=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_MPI=ON" configure_extra_options+=" -DHPX_WITH_PARCELPORT_LCI=ON" configure_extra_options+=" -DHPX_WITH_FETCH_LCI=ON" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET=ON" -configure_extra_options+=" -DHPX_WITH_FETCH_GASNET=ON" configure_extra_options+=" -DCMAKE_C_COMPILER=gcc" configure_extra_options+=" -DCMAKE_C_FLAGS=-fPIC" -configure_extra_options+=" -DHPX_WITH_PARCELPORT_GASNET_CONDUIT=smp" configure_extra_options+=" -DHPX_WITH_DATAPAR_BACKEND=EVE" configure_extra_options+=" -DHPX_WITH_FETCH_EVE=ON" configure_extra_options+=" -DHPX_WITH_EVE_TAG=main" diff --git a/cmake/HPX_AddTest.cmake b/cmake/HPX_AddTest.cmake index 863c003fea9e..cd90dc0d0972 100644 --- a/cmake/HPX_AddTest.cmake +++ b/cmake/HPX_AddTest.cmake @@ -91,14 +91,8 @@ function(add_hpx_test category name) set(Python_EXECUTABLE ${PYTHON_EXECUTABLE}) endif() - set(ENV_VAR "") - if(HPX_WITH_PARCELPORT_GASNET) - set(ENV_VAR "GASNET_PSHM_NODES=2") - endif() - # cmake-format: off set(cmd - ${ENV_VAR} "${Python_EXECUTABLE}" "${_script_location}/bin/hpxrun.py" ${CMAKE_CROSSCOMPILING_EMULATOR} @@ -129,7 +123,7 @@ function(add_hpx_test category name) if(${name}_LOCALITIES STREQUAL "1") set(_full_name "${category}.${name}") - add_test(NAME "${category}.${name}" COMMAND ${cmd} ${args}) + add_test(NAME "${_full_name}" COMMAND ${cmd} ${args}) if(${run_serial}) set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) endif() @@ -180,6 +174,7 @@ function(add_hpx_test category name) ${args} ) set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) + if(${name}_TIMEOUT) set_tests_properties( "${_full_name}" PROPERTIES TIMEOUT ${${name}_TIMEOUT} @@ -201,9 +196,14 @@ function(add_hpx_test category name) if(_add_test) set(_full_name "${category}.distributed.gasnet.${name}") add_test(NAME "${_full_name}" COMMAND ${cmd} "-p" "gasnet" "-r" - "gasnet" ${args} + "gasnet-smp" ${args} + ) + set_tests_properties( + "${_full_name}" + PROPERTIES + RUN_SERIAL TRUE ENVIRONMENT + "PATH=${PROJECT_BINARY_DIR}/_deps/gasnet-src/install/bin:$ENV{PATH}" ) - set_tests_properties("${_full_name}" PROPERTIES RUN_SERIAL TRUE) if(${name}_TIMEOUT) set_tests_properties( "${_full_name}" PROPERTIES TIMEOUT ${${name}_TIMEOUT} diff --git a/cmake/HPX_SetupGasnet.cmake b/cmake/HPX_SetupGasnet.cmake index 26653ac1d944..61690c08ddae 100644 --- a/cmake/HPX_SetupGasnet.cmake +++ b/cmake/HPX_SetupGasnet.cmake @@ -103,11 +103,23 @@ macro(hpx_setup_gasnet) set(GASNET_BUILD_OUTPUT "${GASNET_DIR}/build.log") set(GASNET_ERROR_FILE "${GASNET_DIR}/error.log") + if(CMAKE_BUILD_PARALLEL_LEVEL) + set(GASNET_BUILD_PARALLEL_LEVEL ${CMAKE_BUILD_PARALLEL_LEVEL}) + else() + cmake_host_system_information( + RESULT GASNET_BUILD_PARALLEL_LEVEL QUERY NUMBER_OF_PHYSICAL_CORES + ) + endif() + + if(NOT ${GASNET_BUILD_PARALLEL_LEVEL}) + set(GASNET_BUILD_PARALLEL_LEVEL 1) + endif() + if("${HPX_WITH_PARCELPORT_GASNET_CONDUIT}" STREQUAL "udp") execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC --enable-udp && make && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC --enable-udp && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} @@ -117,7 +129,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC --enable-smp && make && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC --enable-smp && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} @@ -127,7 +139,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-ofi --with-ofi-home=${OFI_DIR} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-ofi --with-ofi-home=${OFI_DIR} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} @@ -137,7 +149,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-ucx --with-ucx-home=${UCX_DIR} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-ucx --with-ucx-home=${UCX_DIR} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} @@ -173,7 +185,7 @@ macro(hpx_setup_gasnet) execute_process( COMMAND bash -c - "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-mpi --with-mpi-cc=${CMAKE_C_COMPILER} --with-mpi-libs=${MPI_C_LIBRARIES} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make && make install" + "CC=${CMAKE_C_COMPILER} CXX=${CMAKE_CXX_COMPILER} CFLAGS=-fPIC CCFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --enable-mpi --with-mpi-cc=${CMAKE_C_COMPILER} --with-mpi-libs=${MPI_C_LIBRARIES} --prefix=${GASNET_DIR}/install --with-cflags=-fPIC --with-cxxflags=-fPIC && make -j ${GASNET_BUILD_PARALLEL_LEVEL} && make install" WORKING_DIRECTORY ${GASNET_DIR} RESULT_VARIABLE GASNET_BUILD_STATUS OUTPUT_FILE ${GASNET_BUILD_OUTPUT} diff --git a/cmake/templates/hpxrun.py.in b/cmake/templates/hpxrun.py.in index 21a5badbd411..3bc50eacd078 100755 --- a/cmake/templates/hpxrun.py.in +++ b/cmake/templates/hpxrun.py.in @@ -195,13 +195,19 @@ def run_mpi(cmd, localities, verbose): print('Executing command: ' + ' '.join(exec_cmd)) subproc(exec_cmd) -# Run with amudprun -# This is executing amudprun with the "-np" option set to the number of localities def run_gasnet(cmd, localities, verbose): - exec_cmd = ['amudprun', '-np', str(localities)] + cmd - if verbose: - print('Executing command: ' + ' '.join(exec_cmd)) - subproc(exec_cmd) + os.environ['GASNET_PSHM_NODES'] = str(localities) + os.environ['GASNET_QUIET'] = 'yes' + os.environ['GASNET_ROUTE_OUTPUT'] = '0' + run_mpi(cmd, localities, verbose) + +def run_gasnet_smp(cmd, localities, verbose): + os.environ['GASNET_PSHM_NODES'] = str(localities) + os.environ['GASNET_BARRIER'] = 'PSHM' + os.environ['GASNET_QUIET'] = 'yes' + os.environ['GASNET_ROUTE_OUTPUT'] = '0' + # No launcher needed + run_none(cmd, localities, nodes, verbose) # Run with srun # This is executing srun with the '-n' option set to the number of localities @@ -229,6 +235,9 @@ def run(cmd, runwrapper, localities, nodes, verbose): if runwrapper == 'gasnet': assert nodes is None, "nodes option only valid with tcp parcelport." run_gasnet(cmd, localities, verbose) + if runwrapper == 'gasnet-smp': + assert nodes is None, "nodes option only valid with smp parcelport." + run_gasnet_smp(cmd, localities, verbose) if runwrapper == 'srun': assert nodes is None, "nodes option only valid with tcp parcelport." run_srun(cmd, localities, verbose) @@ -294,7 +303,7 @@ def check_options(parser, options, args): sys.exit(1) check_valid_runwrapper = (lambda x: - x == 'none' or x == 'mpi' or x == 'srun' or x =='jsrun'); + x == 'none' or x == 'mpi' or x == 'srun' or x =='jsrun' or x == 'gasnet' or x == 'gasnet-smp'); if not check_valid_runwrapper(options.runwrapper): print('Error: Runwrapper option not valid\n', sys.stderr) parser.print_help() @@ -379,7 +388,7 @@ Used by the tcp parcelport only. parser.add_option('-r', '--runwrapper' , action='store', type='string' , dest='runwrapper', default=default_env('HPXRUN_RUNWRAPPER', 'none') - , help='Which runwrapper to use (Options are: none, mpi, gasnet, srun, jsrun) ' + , help='Which runwrapper to use (Options are: none, mpi, gasnet, gasnet-smp, srun, jsrun) ' '(environment variable HPXRUN_RUNWRAPPER)') parser.add_option('-e', '--expected' diff --git a/libs/core/gasnet_base/src/gasnet_environment.cpp b/libs/core/gasnet_base/src/gasnet_environment.cpp index 5c09db35185a..a61a1ad999c9 100644 --- a/libs/core/gasnet_base/src/gasnet_environment.cpp +++ b/libs/core/gasnet_base/src/gasnet_environment.cpp @@ -263,8 +263,8 @@ namespace hpx::util { (get_entry_as(cfg, "hpx.parcel.tcp.priority", 1) > get_entry_as(cfg, "hpx.parcel.gasnet.priority", 0))) || (get_entry_as(cfg, "hpx.parcel.gasnet.enable", 1) && - (get_entry_as(cfg, "hpx.parcel.gasnet.priority", 1) > - get_entry_as(cfg, "hpx.parcel.mpi.priority", 0)))) + (get_entry_as(cfg, "hpx.parcel.mpi.priority", 1) > + get_entry_as(cfg, "hpx.parcel.gasnet.priority", 0)))) { LBT_(info) << "GASNET support disabled via configuration settings\n";