From 18ee7406579fffd9dc1e421e3da7c4766159faa3 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Mon, 6 Jul 2020 03:05:15 -0400 Subject: [PATCH 1/2] avoid using MPI_Rget to make SSC work on Summit --- source/adios2/engine/ssc/SscReader.tcc | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/source/adios2/engine/ssc/SscReader.tcc b/source/adios2/engine/ssc/SscReader.tcc index 8c0356cee6..b8c8ad94ed 100644 --- a/source/adios2/engine/ssc/SscReader.tcc +++ b/source/adios2/engine/ssc/SscReader.tcc @@ -60,16 +60,11 @@ void SscReader::GetDeferredCommon(Variable &variable, totalDataSize += i.second.second; } m_Buffer.resize(totalDataSize); - std::vector requests; for (const auto &i : m_AllReceivingWriterRanks) { - requests.emplace_back(); - MPI_Rget(m_Buffer.data() + i.second.first, i.second.second, - MPI_CHAR, i.first, 0, i.second.second, MPI_CHAR, m_MpiWin, - &requests.back()); + MPI_Get(m_Buffer.data() + i.second.first, i.second.second, MPI_CHAR, + i.first, 0, i.second.second, MPI_CHAR, m_MpiWin); } - MPI_Status statuses[requests.size()]; - MPI_Waitall(requests.size(), requests.data(), statuses); } for (const auto &i : m_AllReceivingWriterRanks) @@ -146,16 +141,11 @@ void SscReader::GetDeferredCommon(Variable &variable, T *data) totalDataSize += i.second.second; } m_Buffer.resize(totalDataSize); - std::vector requests; for (const auto &i : m_AllReceivingWriterRanks) { - requests.emplace_back(); - MPI_Rget(m_Buffer.data() + i.second.first, i.second.second, - MPI_CHAR, i.first, 0, i.second.second, MPI_CHAR, m_MpiWin, - &requests.back()); + MPI_Get(m_Buffer.data() + i.second.first, i.second.second, MPI_CHAR, + i.first, 0, i.second.second, MPI_CHAR, m_MpiWin); } - MPI_Status statuses[requests.size()]; - MPI_Waitall(requests.size(), requests.data(), statuses); } for (const auto &i : m_AllReceivingWriterRanks) From a9854036dc5a8e75ea1d45d37f01a377b1273c26 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Mon, 6 Jul 2020 07:52:20 -0400 Subject: [PATCH 2/2] use MPI_Win_lock and MPI_Win_unlock for arbitrary number of sync point --- source/adios2/engine/ssc/SscReader.cpp | 2 -- source/adios2/engine/ssc/SscReader.tcc | 4 ++++ source/adios2/engine/ssc/SscWriter.cpp | 9 --------- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp index 9fc4b00c0c..8c20632204 100644 --- a/source/adios2/engine/ssc/SscReader.cpp +++ b/source/adios2/engine/ssc/SscReader.cpp @@ -101,7 +101,6 @@ StepStatus SscReader::BeginStep(const StepMode stepMode, m_InitialStep = false; SyncWritePattern(); MPI_Win_create(NULL, 0, 1, MPI_INFO_NULL, m_StreamComm, &m_MpiWin); - MPI_Win_start(m_MpiAllWritersGroup, 0, m_MpiWin); } else { @@ -210,7 +209,6 @@ void SscReader::EndStep() TAU_SCOPED_TIMER_FUNC(); if (m_CurrentStep == 0) { - MPI_Win_complete(m_MpiWin); MPI_Win_free(&m_MpiWin); SyncReadPattern(); MPI_Win_create(m_Buffer.data(), m_Buffer.size(), 1, MPI_INFO_NULL, diff --git a/source/adios2/engine/ssc/SscReader.tcc b/source/adios2/engine/ssc/SscReader.tcc index b8c8ad94ed..d6fb3dcbdc 100644 --- a/source/adios2/engine/ssc/SscReader.tcc +++ b/source/adios2/engine/ssc/SscReader.tcc @@ -62,8 +62,10 @@ void SscReader::GetDeferredCommon(Variable &variable, m_Buffer.resize(totalDataSize); for (const auto &i : m_AllReceivingWriterRanks) { + MPI_Win_lock(MPI_LOCK_SHARED, i.first, 0, m_MpiWin); MPI_Get(m_Buffer.data() + i.second.first, i.second.second, MPI_CHAR, i.first, 0, i.second.second, MPI_CHAR, m_MpiWin); + MPI_Win_unlock(i.first, m_MpiWin); } } @@ -143,8 +145,10 @@ void SscReader::GetDeferredCommon(Variable &variable, T *data) m_Buffer.resize(totalDataSize); for (const auto &i : m_AllReceivingWriterRanks) { + MPI_Win_lock(MPI_LOCK_SHARED, i.first, 0, m_MpiWin); MPI_Get(m_Buffer.data() + i.second.first, i.second.second, MPI_CHAR, i.first, 0, i.second.second, MPI_CHAR, m_MpiWin); + MPI_Win_unlock(i.first, m_MpiWin); } } diff --git a/source/adios2/engine/ssc/SscWriter.cpp b/source/adios2/engine/ssc/SscWriter.cpp index a834392fd3..059c5c1e6d 100644 --- a/source/adios2/engine/ssc/SscWriter.cpp +++ b/source/adios2/engine/ssc/SscWriter.cpp @@ -58,13 +58,6 @@ StepStatus SscWriter::BeginStep(StepMode mode, const float timeoutSeconds) ++m_CurrentStep; } - if (m_Verbosity >= 5) - { - std::cout << "SscWriter::BeginStep, World Rank " << m_StreamRank - << ", Writer Rank " << m_WriterRank << ", Step " - << m_CurrentStep << std::endl; - } - return StepStatus::OK; } @@ -141,8 +134,6 @@ void SscWriter::EndStep() SyncWritePattern(); MPI_Win_create(m_Buffer.data(), m_Buffer.size(), 1, MPI_INFO_NULL, m_StreamComm, &m_MpiWin); - MPI_Win_post(m_MpiAllReadersGroup, 0, m_MpiWin); - MPI_Win_wait(m_MpiWin); MPI_Win_free(&m_MpiWin); SyncReadPattern(); MPI_Win_create(m_Buffer.data(), m_Buffer.size(), 1, MPI_INFO_NULL,