From 1eed67d9c3eee923c8f0723aba73d560fd39a4ee Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Mon, 3 Aug 2020 21:44:27 -0400 Subject: [PATCH 1/3] refined mpi handshake in preparation for adding mpi join for ssc --- source/adios2/engine/ssc/SscReader.cpp | 17 +++++-------- source/adios2/engine/ssc/SscWriter.cpp | 17 +++++-------- source/adios2/helper/adiosMpiHandshake.cpp | 28 ++++++++++++++++++---- source/adios2/helper/adiosMpiHandshake.h | 14 +++++++---- 4 files changed, 46 insertions(+), 30 deletions(-) diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp index b19b29a7f1..9bb5cf5469 100644 --- a/source/adios2/engine/ssc/SscReader.cpp +++ b/source/adios2/engine/ssc/SscReader.cpp @@ -240,19 +240,14 @@ void SscReader::SyncMpiPattern() { TAU_SCOPED_TIMER_FUNC(); - auto appRankMaps = - helper::Handshake(m_Name, 'r', m_OpenTimeoutSecs, CommAsMPI(m_Comm)); - - MPI_Group worldGroup; MPI_Group streamGroup; + MPI_Group readerGroup; + MPI_Comm writerComm; + MPI_Comm readerComm; - MPI_Comm_group(MPI_COMM_WORLD, &worldGroup); - MPI_Group_incl(worldGroup, appRankMaps[1].size(), appRankMaps[1].data(), - &m_MpiAllWritersGroup); - MPI_Group_incl(worldGroup, appRankMaps[0].size(), appRankMaps[0].data(), - &streamGroup); - - MPI_Comm_create_group(MPI_COMM_WORLD, streamGroup, 0, &m_StreamComm); + helper::HandshakeComm(m_Name, 'r', m_OpenTimeoutSecs, CommAsMPI(m_Comm), + streamGroup, m_MpiAllWritersGroup, readerGroup, + m_StreamComm, writerComm, readerComm); } bool SscReader::SyncWritePattern() diff --git a/source/adios2/engine/ssc/SscWriter.cpp b/source/adios2/engine/ssc/SscWriter.cpp index 8cb22a7ce0..88a131533f 100644 --- a/source/adios2/engine/ssc/SscWriter.cpp +++ b/source/adios2/engine/ssc/SscWriter.cpp @@ -186,19 +186,14 @@ void SscWriter::SyncMpiPattern() { TAU_SCOPED_TIMER_FUNC(); - auto appRankMaps = - helper::Handshake(m_Name, 'w', m_OpenTimeoutSecs, CommAsMPI(m_Comm)); - - MPI_Group worldGroup; MPI_Group streamGroup; + MPI_Group writerGroup; + MPI_Comm writerComm; + MPI_Comm readerComm; - MPI_Comm_group(MPI_COMM_WORLD, &worldGroup); - MPI_Group_incl(worldGroup, appRankMaps[2].size(), appRankMaps[2].data(), - &m_MpiAllReadersGroup); - MPI_Group_incl(worldGroup, appRankMaps[0].size(), appRankMaps[0].data(), - &streamGroup); - - MPI_Comm_create_group(MPI_COMM_WORLD, streamGroup, 0, &m_StreamComm); + helper::HandshakeComm(m_Name, 'w', m_OpenTimeoutSecs, CommAsMPI(m_Comm), + streamGroup, writerGroup, m_MpiAllReadersGroup, + m_StreamComm, writerComm, readerComm); } void SscWriter::SyncWritePattern(bool finalStep) diff --git a/source/adios2/helper/adiosMpiHandshake.cpp b/source/adios2/helper/adiosMpiHandshake.cpp index 774e6be232..ef90e5bcc7 100644 --- a/source/adios2/helper/adiosMpiHandshake.cpp +++ b/source/adios2/helper/adiosMpiHandshake.cpp @@ -21,10 +21,30 @@ namespace adios2 namespace helper { -const std::vector> Handshake(const std::string &filename, - const char mode, - const int timeoutSeconds, - MPI_Comm localComm) +void HandshakeComm(const std::string &filename, const char mode, + const int timeoutSeconds, MPI_Comm localComm, + MPI_Group &streamGroup, MPI_Group &writerGroup, + MPI_Group &readerGroup, MPI_Comm &streamComm, + MPI_Comm &writerComm, MPI_Comm &readerComm) +{ + auto appRankMaps = HandshakeRank(filename, mode, timeoutSeconds, localComm); + MPI_Group worldGroup; + MPI_Comm_group(MPI_COMM_WORLD, &worldGroup); + MPI_Group_incl(worldGroup, appRankMaps[0].size(), appRankMaps[0].data(), + &streamGroup); + MPI_Group_incl(worldGroup, appRankMaps[1].size(), appRankMaps[1].data(), + &writerGroup); + MPI_Group_incl(worldGroup, appRankMaps[2].size(), appRankMaps[2].data(), + &readerGroup); + MPI_Comm_create_group(MPI_COMM_WORLD, streamGroup, 0, &streamComm); + MPI_Comm_create_group(MPI_COMM_WORLD, writerGroup, 0, &writerComm); + MPI_Comm_create_group(MPI_COMM_WORLD, readerGroup, 0, &readerComm); +} + +const std::vector> HandshakeRank(const std::string &filename, + const char mode, + const int timeoutSeconds, + MPI_Comm localComm) { std::vector> ret(3); diff --git a/source/adios2/helper/adiosMpiHandshake.h b/source/adios2/helper/adiosMpiHandshake.h index 0f38a4bf5e..7a461fb84f 100644 --- a/source/adios2/helper/adiosMpiHandshake.h +++ b/source/adios2/helper/adiosMpiHandshake.h @@ -48,10 +48,16 @@ namespace helper * for stream *filename*. [1] is the vector of all writer ranks for stream * *filename*. [2] is the vector of all reader ranks for stream *filename*. */ -const std::vector> Handshake(const std::string &filename, - const char mode, - const int timeoutSeconds, - MPI_Comm localComm); +const std::vector> HandshakeRank(const std::string &filename, + const char mode, + const int timeoutSeconds, + MPI_Comm localComm); + +void HandshakeComm(const std::string &filename, const char mode, + const int timeoutSeconds, MPI_Comm localComm, + MPI_Group &streamGroup, MPI_Group &writerGroup, + MPI_Group &readerGroup, MPI_Comm &streamComm, + MPI_Comm &writerComm, MPI_Comm &readerComm); } // end namespace helper } // end namespace adios2 From 905358a3fb3bd35bfbfefe4bbb46109f941e30ca Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Mon, 3 Aug 2020 23:20:39 -0400 Subject: [PATCH 2/3] fixing windows warnings --- source/adios2/helper/adiosMpiHandshake.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/adios2/helper/adiosMpiHandshake.cpp b/source/adios2/helper/adiosMpiHandshake.cpp index ef90e5bcc7..d4fd51b3d5 100644 --- a/source/adios2/helper/adiosMpiHandshake.cpp +++ b/source/adios2/helper/adiosMpiHandshake.cpp @@ -30,12 +30,12 @@ void HandshakeComm(const std::string &filename, const char mode, auto appRankMaps = HandshakeRank(filename, mode, timeoutSeconds, localComm); MPI_Group worldGroup; MPI_Comm_group(MPI_COMM_WORLD, &worldGroup); - MPI_Group_incl(worldGroup, appRankMaps[0].size(), appRankMaps[0].data(), - &streamGroup); - MPI_Group_incl(worldGroup, appRankMaps[1].size(), appRankMaps[1].data(), - &writerGroup); - MPI_Group_incl(worldGroup, appRankMaps[2].size(), appRankMaps[2].data(), - &readerGroup); + MPI_Group_incl(worldGroup, static_cast(appRankMaps[0].size()), + appRankMaps[0].data(), &streamGroup); + MPI_Group_incl(worldGroup, static_cast(appRankMaps[1].size()), + appRankMaps[1].data(), &writerGroup); + MPI_Group_incl(worldGroup, static_cast(appRankMaps[2].size()), + appRankMaps[2].data(), &readerGroup); MPI_Comm_create_group(MPI_COMM_WORLD, streamGroup, 0, &streamComm); MPI_Comm_create_group(MPI_COMM_WORLD, writerGroup, 0, &writerComm); MPI_Comm_create_group(MPI_COMM_WORLD, readerGroup, 0, &readerComm); From c0580983edc0e2cd80972853e5e987c43e2093e7 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Mon, 3 Aug 2020 23:41:37 -0400 Subject: [PATCH 3/3] disable windows for mpi handshake because MPI_Comm_create_group does not work on windows --- source/adios2/helper/adiosMpiHandshake.cpp | 2 ++ source/adios2/helper/adiosMpiHandshake.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/source/adios2/helper/adiosMpiHandshake.cpp b/source/adios2/helper/adiosMpiHandshake.cpp index d4fd51b3d5..bc36a88dd9 100644 --- a/source/adios2/helper/adiosMpiHandshake.cpp +++ b/source/adios2/helper/adiosMpiHandshake.cpp @@ -21,6 +21,7 @@ namespace adios2 namespace helper { +#ifndef _WIN32 void HandshakeComm(const std::string &filename, const char mode, const int timeoutSeconds, MPI_Comm localComm, MPI_Group &streamGroup, MPI_Group &writerGroup, @@ -40,6 +41,7 @@ void HandshakeComm(const std::string &filename, const char mode, MPI_Comm_create_group(MPI_COMM_WORLD, writerGroup, 0, &writerComm); MPI_Comm_create_group(MPI_COMM_WORLD, readerGroup, 0, &readerComm); } +#endif const std::vector> HandshakeRank(const std::string &filename, const char mode, diff --git a/source/adios2/helper/adiosMpiHandshake.h b/source/adios2/helper/adiosMpiHandshake.h index 7a461fb84f..84e0ee6a38 100644 --- a/source/adios2/helper/adiosMpiHandshake.h +++ b/source/adios2/helper/adiosMpiHandshake.h @@ -53,11 +53,13 @@ const std::vector> HandshakeRank(const std::string &filename, const int timeoutSeconds, MPI_Comm localComm); +#ifndef _WIN32 void HandshakeComm(const std::string &filename, const char mode, const int timeoutSeconds, MPI_Comm localComm, MPI_Group &streamGroup, MPI_Group &writerGroup, MPI_Group &readerGroup, MPI_Comm &streamComm, MPI_Comm &writerComm, MPI_Comm &readerComm); +#endif } // end namespace helper } // end namespace adios2