From 4bcd66547b66bd1c4ce09909cfc3fb83f0ed0f43 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Wed, 9 Sep 2020 20:16:50 -0400 Subject: [PATCH 1/2] auto call BeginStep in SscReader::Close --- source/adios2/engine/ssc/SscReader.cpp | 11 + source/adios2/engine/ssc/SscReader.h | 3 +- source/adios2/engine/ssc/SscWriter.h | 2 +- testing/adios2/engine/ssc/CMakeLists.txt | 3 + .../engine/ssc/TestSscXgc3WayMatchedSteps.cpp | 300 ++++++++++++++++++ 5 files changed, 317 insertions(+), 2 deletions(-) create mode 100644 testing/adios2/engine/ssc/TestSscXgc3WayMatchedSteps.cpp diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp index 85ad434f36..0e0712a5ba 100644 --- a/source/adios2/engine/ssc/SscReader.cpp +++ b/source/adios2/engine/ssc/SscReader.cpp @@ -50,6 +50,8 @@ StepStatus SscReader::BeginStep(const StepMode stepMode, ++m_CurrentStep; + m_StepBegun = true; + if (m_Verbosity >= 5) { std::cout << "SscReader::BeginStep, World Rank " << m_StreamRank @@ -233,6 +235,9 @@ void SscReader::EndStep() SyncReadPattern(); } } + + m_StepBegun = false; + } // PRIVATE @@ -522,6 +527,12 @@ void SscReader::DoClose(const int transportIndex) std::cout << "SscReader::DoClose, World Rank " << m_StreamRank << ", Reader Rank " << m_ReaderRank << std::endl; } + + if(!m_StepBegun) + { + BeginStep(); + } + if (m_WriterDefinitionsLocked && m_ReaderSelectionsLocked) { MPI_Win_free(&m_MpiWin); diff --git a/source/adios2/engine/ssc/SscReader.h b/source/adios2/engine/ssc/SscReader.h index b5941dd0e7..d7f5ceec5d 100644 --- a/source/adios2/engine/ssc/SscReader.h +++ b/source/adios2/engine/ssc/SscReader.h @@ -41,6 +41,7 @@ class SscReader : public Engine private: int64_t m_CurrentStep = -1; + bool m_StepBegun = false; ssc::BlockVecVec m_GlobalWritePattern; ssc::BlockVec m_LocalReadPattern; @@ -85,7 +86,7 @@ class SscReader : public Engine void CalculatePosition(ssc::BlockVecVec &mapVec, ssc::RankPosMap &allOverlapRanks); - int m_Verbosity = 0; + int m_Verbosity = 10; int m_OpenTimeoutSecs = 10; }; diff --git a/source/adios2/engine/ssc/SscWriter.h b/source/adios2/engine/ssc/SscWriter.h index 1866f3c0b7..25d5d0d5bc 100644 --- a/source/adios2/engine/ssc/SscWriter.h +++ b/source/adios2/engine/ssc/SscWriter.h @@ -80,7 +80,7 @@ class SscWriter : public Engine ssc::BlockVecVec &readerMapVec, const int writerRank, ssc::RankPosMap &allOverlapRanks); - int m_Verbosity = 0; + int m_Verbosity = 10; int m_OpenTimeoutSecs = 10; }; diff --git a/testing/adios2/engine/ssc/CMakeLists.txt b/testing/adios2/engine/ssc/CMakeLists.txt index ceb4b8f1f2..3453f837a1 100644 --- a/testing/adios2/engine/ssc/CMakeLists.txt +++ b/testing/adios2/engine/ssc/CMakeLists.txt @@ -63,6 +63,9 @@ if(ADIOS2_HAVE_MPI) if(NOT MSVC) gtest_add_tests_helper(Xgc3Way MPI_ONLY Ssc Engine.SSC. "") SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscXgc3Way.MPI "" TRUE) + + gtest_add_tests_helper(Xgc3WayMatchedSteps MPI_ONLY Ssc Engine.SSC. "") + SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscXgc3WayMatchedSteps.MPI "" TRUE) endif() gtest_add_tests_helper(VaryingSteps MPI_ONLY Ssc Engine.SSC. "") diff --git a/testing/adios2/engine/ssc/TestSscXgc3WayMatchedSteps.cpp b/testing/adios2/engine/ssc/TestSscXgc3WayMatchedSteps.cpp new file mode 100644 index 0000000000..9442249023 --- /dev/null +++ b/testing/adios2/engine/ssc/TestSscXgc3WayMatchedSteps.cpp @@ -0,0 +1,300 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ + +#include "TestSscCommon.h" +#include +#include +#include +#include +#include + +using namespace adios2; +int mpiRank = 0; +int mpiSize = 1; +int mpiGroup; +MPI_Comm mpiComm; + +class SscEngineTest : public ::testing::Test +{ +public: + SscEngineTest() = default; +}; + +void coupler(const Dims &shape, const Dims &start, const Dims &count, + const size_t steps, const adios2::Params &engineParams) +{ + size_t datasize = + std::accumulate(count.begin(), count.end(), static_cast(1), + std::multiplies()); + + adios2::ADIOS adios(mpiComm); + + adios2::IO x_to_c_io = adios.DeclareIO("x_to_c"); + x_to_c_io.SetEngine("ssc"); + x_to_c_io.SetParameters(engineParams); + + adios2::IO c_to_x_io = adios.DeclareIO("c_to_x"); + c_to_x_io.SetEngine("ssc"); + c_to_x_io.SetParameters(engineParams); + + adios2::IO g_to_c_io = adios.DeclareIO("g_to_c"); + g_to_c_io.SetEngine("ssc"); + g_to_c_io.SetParameters(engineParams); + + adios2::IO c_to_g_io = adios.DeclareIO("c_to_g"); + c_to_g_io.SetEngine("ssc"); + c_to_g_io.SetParameters(engineParams); + + std::vector x_to_c_data; + std::vector c_to_x_data(datasize); + std::vector g_to_c_data; + std::vector c_to_g_data(datasize); + + auto c_to_x_var = + c_to_x_io.DefineVariable("c_to_x", shape, start, count); + auto c_to_g_var = + c_to_g_io.DefineVariable("c_to_g", shape, start, count); + + adios2::Engine x_to_c_engine = x_to_c_io.Open("x_to_c", adios2::Mode::Read); + x_to_c_engine.LockReaderSelections(); + + adios2::Engine c_to_x_engine = + c_to_x_io.Open("c_to_x", adios2::Mode::Write); + c_to_x_engine.LockWriterDefinitions(); + + adios2::Engine c_to_g_engine = + c_to_g_io.Open("c_to_g", adios2::Mode::Write); + c_to_g_engine.LockWriterDefinitions(); + + adios2::Engine g_to_c_engine = g_to_c_io.Open("g_to_c", adios2::Mode::Read); + g_to_c_engine.LockReaderSelections(); + + for (int i = 0; i < steps; ++i) + { + x_to_c_engine.BeginStep(); + auto x_to_c_var = x_to_c_io.InquireVariable("x_to_c"); + auto readShape = x_to_c_var.Shape(); + x_to_c_data.resize(std::accumulate(readShape.begin(), readShape.end(), + static_cast(1), + std::multiplies())); + x_to_c_engine.Get(x_to_c_var, x_to_c_data.data(), adios2::Mode::Sync); + VerifyData(x_to_c_data.data(), i, Dims(readShape.size(), 0), readShape, + readShape, mpiRank); + x_to_c_engine.EndStep(); + + c_to_g_engine.BeginStep(); + GenData(c_to_g_data, i, start, count, shape); + c_to_g_engine.Put(c_to_g_var, c_to_g_data.data(), adios2::Mode::Sync); + c_to_g_engine.EndStep(); + + g_to_c_engine.BeginStep(); + auto g_to_c_var = g_to_c_io.InquireVariable("g_to_c"); + readShape = g_to_c_var.Shape(); + g_to_c_data.resize(std::accumulate(readShape.begin(), readShape.end(), + static_cast(1), + std::multiplies())); + g_to_c_engine.Get(g_to_c_var, g_to_c_data.data(), adios2::Mode::Sync); + VerifyData(g_to_c_data.data(), i, Dims(readShape.size(), 0), readShape, + readShape, mpiRank); + g_to_c_engine.EndStep(); + + c_to_x_engine.BeginStep(); + GenData(c_to_x_data, i, start, count, shape); + c_to_x_engine.Put(c_to_x_var, c_to_x_data.data(), adios2::Mode::Sync); + c_to_x_engine.EndStep(); + } + + x_to_c_engine.Close(); + c_to_g_engine.Close(); + g_to_c_engine.Close(); + c_to_x_engine.Close(); +} + +void xgc(const Dims &shape, const Dims &start, const Dims &count, + const size_t steps, const adios2::Params &engineParams) +{ + size_t datasize = + std::accumulate(count.begin(), count.end(), static_cast(1), + std::multiplies()); + + std::vector x_to_c_data(datasize); + std::vector c_to_x_data; + + adios2::ADIOS adios(mpiComm); + + adios2::IO x_to_c_io = adios.DeclareIO("x_to_c"); + x_to_c_io.SetEngine("ssc"); + x_to_c_io.SetParameters(engineParams); + + adios2::IO c_to_x_io = adios.DeclareIO("c_to_x"); + c_to_x_io.SetEngine("ssc"); + c_to_x_io.SetParameters(engineParams); + + auto x_to_c_var = + x_to_c_io.DefineVariable("x_to_c", shape, start, count); + + adios2::Engine x_to_c_engine = + x_to_c_io.Open("x_to_c", adios2::Mode::Write); + x_to_c_engine.LockWriterDefinitions(); + + adios2::Engine c_to_x_engine = c_to_x_io.Open("c_to_x", adios2::Mode::Read); + c_to_x_engine.LockReaderSelections(); + + for (int i = 0; i < steps; ++i) + { + x_to_c_engine.BeginStep(); + GenData(x_to_c_data, i, start, count, shape); + x_to_c_engine.Put(x_to_c_var, x_to_c_data.data(), adios2::Mode::Sync); + x_to_c_engine.EndStep(); + + c_to_x_engine.BeginStep(); + auto c_to_x_var = c_to_x_io.InquireVariable("c_to_x"); + auto readShape = c_to_x_var.Shape(); + c_to_x_data.resize(std::accumulate(readShape.begin(), readShape.end(), + static_cast(1), + std::multiplies())); + c_to_x_engine.Get(c_to_x_var, c_to_x_data.data(), adios2::Mode::Sync); + VerifyData(c_to_x_data.data(), i, Dims(readShape.size(), 0), readShape, + readShape, mpiRank); + c_to_x_engine.EndStep(); + } + + x_to_c_engine.Close(); + c_to_x_engine.Close(); +} + +void gene(const Dims &shape, const Dims &start, const Dims &count, + const size_t steps, const adios2::Params &engineParams) +{ + adios2::ADIOS adios(mpiComm); + + adios2::IO c_to_g_io = adios.DeclareIO("c_to_g"); + c_to_g_io.SetEngine("ssc"); + c_to_g_io.SetParameters(engineParams); + + adios2::IO g_to_c_io = adios.DeclareIO("g_to_c"); + g_to_c_io.SetEngine("ssc"); + g_to_c_io.SetParameters(engineParams); + + auto g_to_c_var = + g_to_c_io.DefineVariable("g_to_c", shape, start, count); + + adios2::Engine c_to_g_engine = c_to_g_io.Open("c_to_g", adios2::Mode::Read); + c_to_g_engine.LockReaderSelections(); + adios2::Engine g_to_c_engine = + g_to_c_io.Open("g_to_c", adios2::Mode::Write); + g_to_c_engine.LockWriterDefinitions(); + + size_t datasize = + std::accumulate(shape.begin(), shape.end(), static_cast(1), + std::multiplies()); + std::vector c_to_g_data; + std::vector g_to_c_data(datasize); + + for (int i = 0; i < steps; ++i) + { + c_to_g_engine.BeginStep(StepMode::Read, 5); + auto c_to_g_var = c_to_g_io.InquireVariable("c_to_g"); + auto readShape = c_to_g_var.Shape(); + c_to_g_data.resize(std::accumulate(readShape.begin(), readShape.end(), + static_cast(1), + std::multiplies())); + c_to_g_engine.Get(c_to_g_var, c_to_g_data.data(), adios2::Mode::Sync); + VerifyData(c_to_g_data.data(), i, Dims(readShape.size(), 0), readShape, + readShape, mpiRank); + c_to_g_engine.EndStep(); + + g_to_c_engine.BeginStep(); + GenData(g_to_c_data, i, start, count, shape); + g_to_c_engine.Put(g_to_c_var, g_to_c_data.data(), adios2::Mode::Sync); + g_to_c_engine.EndStep(); + } + + c_to_g_engine.Close(); + g_to_c_engine.Close(); +} + +TEST_F(SscEngineTest, TestSscXgc3WayMatchedSteps) +{ + + Dims start, count, shape; + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + + if (worldSize < 4) + { + return; + } + + if (worldRank == 0) + { + mpiGroup = 0; + } + else if (worldRank == 1) + { + mpiGroup = 1; + } + else + { + mpiGroup = 2; + } + + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + size_t steps = 20; + + if (mpiGroup == 0) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"Verbose", "0"}}; + coupler(shape, start, count, steps, engineParams); + } + + if (mpiGroup == 1) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"Verbose", "0"}}; + gene(shape, start, shape, steps, engineParams); + } + + if (mpiGroup == 2) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"Verbose", "0"}}; + xgc(shape, start, count, steps, engineParams); + } + + MPI_Barrier(MPI_COMM_WORLD); +} + +int main(int argc, char **argv) +{ + MPI_Init(&argc, &argv); + + ::testing::InitGoogleTest(&argc, argv); + int result = RUN_ALL_TESTS(); + + MPI_Finalize(); + return result; +} From 9aa62778c80e7353b8eaa8b368b6b4316a47dea5 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Wed, 9 Sep 2020 20:19:43 -0400 Subject: [PATCH 2/2] clang-format --- source/adios2/engine/ssc/SscReader.cpp | 3 +-- source/adios2/engine/ssc/SscReader.h | 2 +- source/adios2/engine/ssc/SscWriter.h | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp index 0e0712a5ba..16074f7883 100644 --- a/source/adios2/engine/ssc/SscReader.cpp +++ b/source/adios2/engine/ssc/SscReader.cpp @@ -237,7 +237,6 @@ void SscReader::EndStep() } m_StepBegun = false; - } // PRIVATE @@ -528,7 +527,7 @@ void SscReader::DoClose(const int transportIndex) << ", Reader Rank " << m_ReaderRank << std::endl; } - if(!m_StepBegun) + if (!m_StepBegun) { BeginStep(); } diff --git a/source/adios2/engine/ssc/SscReader.h b/source/adios2/engine/ssc/SscReader.h index d7f5ceec5d..e5c34adf15 100644 --- a/source/adios2/engine/ssc/SscReader.h +++ b/source/adios2/engine/ssc/SscReader.h @@ -86,7 +86,7 @@ class SscReader : public Engine void CalculatePosition(ssc::BlockVecVec &mapVec, ssc::RankPosMap &allOverlapRanks); - int m_Verbosity = 10; + int m_Verbosity = 0; int m_OpenTimeoutSecs = 10; }; diff --git a/source/adios2/engine/ssc/SscWriter.h b/source/adios2/engine/ssc/SscWriter.h index 25d5d0d5bc..1866f3c0b7 100644 --- a/source/adios2/engine/ssc/SscWriter.h +++ b/source/adios2/engine/ssc/SscWriter.h @@ -80,7 +80,7 @@ class SscWriter : public Engine ssc::BlockVecVec &readerMapVec, const int writerRank, ssc::RankPosMap &allOverlapRanks); - int m_Verbosity = 10; + int m_Verbosity = 0; int m_OpenTimeoutSecs = 10; };