diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp index 8c20632204..949db4b6dd 100644 --- a/source/adios2/engine/ssc/SscReader.cpp +++ b/source/adios2/engine/ssc/SscReader.cpp @@ -46,51 +46,6 @@ SscReader::SscReader(IO &io, const std::string &name, const Mode mode, SscReader::~SscReader() { TAU_SCOPED_TIMER_FUNC(); } -void SscReader::GetOneSidedPostPush() -{ - TAU_SCOPED_TIMER_FUNC(); - MPI_Win_post(m_MpiAllWritersGroup, 0, m_MpiWin); -} - -void SscReader::GetOneSidedFencePush() -{ - TAU_SCOPED_TIMER_FUNC(); - MPI_Win_fence(0, m_MpiWin); -} - -void SscReader::GetOneSidedPostPull() -{ - TAU_SCOPED_TIMER_FUNC(); - MPI_Win_start(m_MpiAllWritersGroup, 0, m_MpiWin); - for (const auto &i : m_AllReceivingWriterRanks) - { - MPI_Get(m_Buffer.data() + i.second.first, i.second.second, MPI_CHAR, - i.first, 0, i.second.second, MPI_CHAR, m_MpiWin); - } -} - -void SscReader::GetOneSidedFencePull() -{ - TAU_SCOPED_TIMER_FUNC(); - MPI_Win_fence(0, m_MpiWin); - for (const auto &i : m_AllReceivingWriterRanks) - { - MPI_Get(m_Buffer.data() + i.second.first, i.second.second, MPI_CHAR, - i.first, 0, i.second.second, MPI_CHAR, m_MpiWin); - } -} - -void SscReader::GetTwoSided() -{ - TAU_SCOPED_TIMER_FUNC(); - for (const auto &i : m_AllReceivingWriterRanks) - { - m_MpiRequests.emplace_back(); - MPI_Irecv(m_Buffer.data() + i.second.first, i.second.second, MPI_CHAR, - i.first, 0, m_StreamComm, &m_MpiRequests.back()); - } -} - StepStatus SscReader::BeginStep(const StepMode stepMode, const float timeoutSeconds) { @@ -217,23 +172,39 @@ void SscReader::EndStep() if (m_MpiMode == "twosided") { - GetTwoSided(); + for (const auto &i : m_AllReceivingWriterRanks) + { + m_MpiRequests.emplace_back(); + MPI_Irecv(m_Buffer.data() + i.second.first, i.second.second, + MPI_CHAR, i.first, 0, m_StreamComm, + &m_MpiRequests.back()); + } } else if (m_MpiMode == "onesidedfencepush") { - GetOneSidedFencePush(); + MPI_Win_fence(0, m_MpiWin); } else if (m_MpiMode == "onesidedpostpush") { - GetOneSidedPostPush(); + MPI_Win_post(m_MpiAllWritersGroup, 0, m_MpiWin); } else if (m_MpiMode == "onesidedfencepull") { - GetOneSidedFencePull(); + MPI_Win_fence(0, m_MpiWin); + for (const auto &i : m_AllReceivingWriterRanks) + { + MPI_Get(m_Buffer.data() + i.second.first, i.second.second, MPI_CHAR, + i.first, 0, i.second.second, MPI_CHAR, m_MpiWin); + } } else if (m_MpiMode == "onesidedpostpull") { - GetOneSidedPostPull(); + MPI_Win_start(m_MpiAllWritersGroup, 0, m_MpiWin); + for (const auto &i : m_AllReceivingWriterRanks) + { + MPI_Get(m_Buffer.data() + i.second.first, i.second.second, MPI_CHAR, + i.first, 0, i.second.second, MPI_CHAR, m_MpiWin); + } } } diff --git a/source/adios2/engine/ssc/SscReader.h b/source/adios2/engine/ssc/SscReader.h index 1c4feb27a4..11dba7cbf0 100644 --- a/source/adios2/engine/ssc/SscReader.h +++ b/source/adios2/engine/ssc/SscReader.h @@ -63,11 +63,6 @@ class SscReader : public Engine void SyncMpiPattern(); void SyncWritePattern(); void SyncReadPattern(); - void GetOneSidedFencePush(); - void GetOneSidedPostPush(); - void GetOneSidedFencePull(); - void GetOneSidedPostPull(); - void GetTwoSided(); #define declare_type(T) \ void DoGetSync(Variable &, T *) final; \ diff --git a/source/adios2/engine/ssc/SscWriter.cpp b/source/adios2/engine/ssc/SscWriter.cpp index 059c5c1e6d..c931a57459 100644 --- a/source/adios2/engine/ssc/SscWriter.cpp +++ b/source/adios2/engine/ssc/SscWriter.cpp @@ -47,8 +47,6 @@ StepStatus SscWriter::BeginStep(StepMode mode, const float timeoutSeconds) { TAU_SCOPED_TIMER_FUNC(); - MpiWait(); - if (m_InitialStep) { m_InitialStep = false; @@ -58,6 +56,11 @@ StepStatus SscWriter::BeginStep(StepMode mode, const float timeoutSeconds) ++m_CurrentStep; } + if (m_CurrentStep > 1) + { + MpiWait(); + } + return StepStatus::OK; } @@ -69,56 +72,6 @@ size_t SscWriter::CurrentStep() const void SscWriter::PerformPuts() { TAU_SCOPED_TIMER_FUNC(); } -void SscWriter::PutOneSidedPostPush() -{ - TAU_SCOPED_TIMER_FUNC(); - MPI_Win_start(m_MpiAllReadersGroup, 0, m_MpiWin); - for (const auto &i : m_AllSendingReaderRanks) - { - MPI_Put(m_Buffer.data(), m_Buffer.size(), MPI_CHAR, i.first, - i.second.first, m_Buffer.size(), MPI_CHAR, m_MpiWin); - } - m_NeedWait = true; -} - -void SscWriter::PutOneSidedFencePush() -{ - TAU_SCOPED_TIMER_FUNC(); - MPI_Win_fence(0, m_MpiWin); - for (const auto &i : m_AllSendingReaderRanks) - { - MPI_Put(m_Buffer.data(), m_Buffer.size(), MPI_CHAR, i.first, - i.second.first, m_Buffer.size(), MPI_CHAR, m_MpiWin); - } - m_NeedWait = true; -} - -void SscWriter::PutOneSidedPostPull() -{ - TAU_SCOPED_TIMER_FUNC(); - MPI_Win_post(m_MpiAllReadersGroup, 0, m_MpiWin); - m_NeedWait = true; -} - -void SscWriter::PutOneSidedFencePull() -{ - TAU_SCOPED_TIMER_FUNC(); - MPI_Win_fence(0, m_MpiWin); - m_NeedWait = true; -} - -void SscWriter::PutTwoSided() -{ - TAU_SCOPED_TIMER_FUNC(); - for (const auto &i : m_AllSendingReaderRanks) - { - m_MpiRequests.emplace_back(); - MPI_Isend(m_Buffer.data(), m_Buffer.size(), MPI_CHAR, i.first, 0, - m_StreamComm, &m_MpiRequests.back()); - } - m_NeedWait = true; -} - void SscWriter::EndStep() { TAU_SCOPED_TIMER_FUNC(); @@ -143,23 +96,38 @@ void SscWriter::EndStep() { if (m_MpiMode == "twosided") { - PutTwoSided(); + for (const auto &i : m_AllSendingReaderRanks) + { + m_MpiRequests.emplace_back(); + MPI_Isend(m_Buffer.data(), m_Buffer.size(), MPI_CHAR, i.first, + 0, m_StreamComm, &m_MpiRequests.back()); + } } else if (m_MpiMode == "onesidedfencepush") { - PutOneSidedFencePush(); + MPI_Win_fence(0, m_MpiWin); + for (const auto &i : m_AllSendingReaderRanks) + { + MPI_Put(m_Buffer.data(), m_Buffer.size(), MPI_CHAR, i.first, + i.second.first, m_Buffer.size(), MPI_CHAR, m_MpiWin); + } } else if (m_MpiMode == "onesidedpostpush") { - PutOneSidedPostPush(); + MPI_Win_start(m_MpiAllReadersGroup, 0, m_MpiWin); + for (const auto &i : m_AllSendingReaderRanks) + { + MPI_Put(m_Buffer.data(), m_Buffer.size(), MPI_CHAR, i.first, + i.second.first, m_Buffer.size(), MPI_CHAR, m_MpiWin); + } } else if (m_MpiMode == "onesidedfencepull") { - PutOneSidedFencePull(); + MPI_Win_fence(0, m_MpiWin); } else if (m_MpiMode == "onesidedpostpull") { - PutOneSidedPostPull(); + MPI_Win_post(m_MpiAllReadersGroup, 0, m_MpiWin); } } } @@ -170,31 +138,27 @@ void SscWriter::Flush(const int transportIndex) { TAU_SCOPED_TIMER_FUNC(); } void SscWriter::MpiWait() { - if (m_NeedWait) + if (m_MpiMode == "twosided") { - if (m_MpiMode == "twosided") - { - MPI_Status statuses[m_MpiRequests.size()]; - MPI_Waitall(m_MpiRequests.size(), m_MpiRequests.data(), statuses); - m_MpiRequests.clear(); - } - else if (m_MpiMode == "onesidedfencepush") - { - MPI_Win_fence(0, m_MpiWin); - } - else if (m_MpiMode == "onesidedpostpush") - { - MPI_Win_complete(m_MpiWin); - } - else if (m_MpiMode == "onesidedfencepull") - { - MPI_Win_fence(0, m_MpiWin); - } - else if (m_MpiMode == "onesidedpostpull") - { - MPI_Win_wait(m_MpiWin); - } - m_NeedWait = false; + MPI_Status statuses[m_MpiRequests.size()]; + MPI_Waitall(m_MpiRequests.size(), m_MpiRequests.data(), statuses); + m_MpiRequests.clear(); + } + else if (m_MpiMode == "onesidedfencepush") + { + MPI_Win_fence(0, m_MpiWin); + } + else if (m_MpiMode == "onesidedpostpush") + { + MPI_Win_complete(m_MpiWin); + } + else if (m_MpiMode == "onesidedfencepull") + { + MPI_Win_fence(0, m_MpiWin); + } + else if (m_MpiMode == "onesidedpostpull") + { + MPI_Win_wait(m_MpiWin); } } @@ -369,7 +333,10 @@ void SscWriter::DoClose(const int transportIndex) { TAU_SCOPED_TIMER_FUNC(); - MpiWait(); + if (m_CurrentStep > 0) + { + MpiWait(); + } m_Buffer[0] = 1; diff --git a/source/adios2/engine/ssc/SscWriter.h b/source/adios2/engine/ssc/SscWriter.h index 9ffc503746..6d7a61261c 100644 --- a/source/adios2/engine/ssc/SscWriter.h +++ b/source/adios2/engine/ssc/SscWriter.h @@ -54,7 +54,6 @@ class SscWriter : public Engine MPI_Group m_MpiAllReadersGroup; MPI_Comm m_StreamComm; std::string m_MpiMode = "twosided"; - bool m_NeedWait = false; std::vector m_MpiRequests; int m_StreamRank; @@ -65,11 +64,6 @@ class SscWriter : public Engine void SyncMpiPattern(); void SyncWritePattern(); void SyncReadPattern(); - void PutOneSidedFencePush(); - void PutOneSidedPostPush(); - void PutOneSidedFencePull(); - void PutOneSidedPostPull(); - void PutTwoSided(); void MpiWait(); #define declare_type(T) \ diff --git a/testing/adios2/engine/ssc/CMakeLists.txt b/testing/adios2/engine/ssc/CMakeLists.txt index 9f7b008819..38fb9f9015 100644 --- a/testing/adios2/engine/ssc/CMakeLists.txt +++ b/testing/adios2/engine/ssc/CMakeLists.txt @@ -9,6 +9,12 @@ if(ADIOS2_HAVE_MPI) gtest_add_tests_helper(Base MPI_ONLY Ssc Engine.SSC. "") SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscBase.MPI "" TRUE) + gtest_add_tests_helper(OnlyOneStep MPI_ONLY Ssc Engine.SSC. "") + SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscOnlyOneStep.MPI "" TRUE) + + gtest_add_tests_helper(OnlyTwoSteps MPI_ONLY Ssc Engine.SSC. "") + SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscOnlyTwoSteps.MPI "" TRUE) + gtest_add_tests_helper(OneSidedFencePush MPI_ONLY Ssc Engine.SSC. "") SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscOneSidedFencePush.MPI "" TRUE) diff --git a/testing/adios2/engine/ssc/TestSscOnlyOneStep.cpp b/testing/adios2/engine/ssc/TestSscOnlyOneStep.cpp new file mode 100644 index 0000000000..598cce88b5 --- /dev/null +++ b/testing/adios2/engine/ssc/TestSscOnlyOneStep.cpp @@ -0,0 +1,283 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ + +#include "TestSscCommon.h" +#include +#include +#include +#include +#include + +using namespace adios2; +int mpiRank = 0; +int mpiSize = 1; +MPI_Comm mpiComm; + +class SscEngineTest : public ::testing::Test +{ +public: + SscEngineTest() = default; +}; + +void Writer(const Dims &shape, const Dims &start, const Dims &count, + const size_t steps, const adios2::Params &engineParams, + const std::string &name) +{ + size_t datasize = std::accumulate(count.begin(), count.end(), 1, + std::multiplies()); + adios2::ADIOS adios(mpiComm); + adios2::IO dataManIO = adios.DeclareIO("WAN"); + dataManIO.SetEngine("ssc"); + dataManIO.SetParameters(engineParams); + std::vector myChars(datasize); + std::vector myUChars(datasize); + std::vector myShorts(datasize); + std::vector myUShorts(datasize); + std::vector myInts(datasize); + std::vector myUInts(datasize); + std::vector myFloats(datasize); + std::vector myDoubles(datasize); + std::vector> myComplexes(datasize); + std::vector> myDComplexes(datasize); + auto bpChars = + dataManIO.DefineVariable("bpChars", shape, start, count); + auto bpUChars = dataManIO.DefineVariable("bpUChars", shape, + start, count); + auto bpShorts = + dataManIO.DefineVariable("bpShorts", shape, start, count); + auto bpUShorts = dataManIO.DefineVariable( + "bpUShorts", shape, start, count); + auto bpInts = dataManIO.DefineVariable("bpInts", shape, start, count); + auto bpUInts = + dataManIO.DefineVariable("bpUInts", shape, start, count); + auto bpFloats = + dataManIO.DefineVariable("bpFloats", shape, start, count); + auto bpDoubles = + dataManIO.DefineVariable("bpDoubles", shape, start, count); + auto bpComplexes = dataManIO.DefineVariable>( + "bpComplexes", shape, start, count); + auto bpDComplexes = dataManIO.DefineVariable>( + "bpDComplexes", shape, start, count); + auto scalarInt = dataManIO.DefineVariable("scalarInt"); + auto stringVar = dataManIO.DefineVariable("stringVar"); + dataManIO.DefineAttribute("AttInt", 110); + adios2::Engine dataManWriter = dataManIO.Open(name, adios2::Mode::Write); + for (int i = 0; i < steps; ++i) + { + dataManWriter.BeginStep(); + GenData(myChars, i, start, count, shape); + GenData(myUChars, i, start, count, shape); + GenData(myShorts, i, start, count, shape); + GenData(myUShorts, i, start, count, shape); + GenData(myInts, i, start, count, shape); + GenData(myUInts, i, start, count, shape); + GenData(myFloats, i, start, count, shape); + GenData(myDoubles, i, start, count, shape); + GenData(myComplexes, i, start, count, shape); + GenData(myDComplexes, i, start, count, shape); + dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync); + dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync); + dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync); + dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync); + dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync); + dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync); + dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync); + dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync); + dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync); + dataManWriter.Put(bpDComplexes, myDComplexes.data(), + adios2::Mode::Sync); + dataManWriter.Put(scalarInt, i); + std::string s = "sample string sample string sample string"; + dataManWriter.Put(stringVar, s); + dataManWriter.EndStep(); + } + dataManWriter.Close(); +} + +void Reader(const Dims &shape, const Dims &start, const Dims &count, + const size_t steps, const adios2::Params &engineParams, + const std::string &name) +{ + adios2::ADIOS adios(mpiComm); + adios2::IO dataManIO = adios.DeclareIO("Test"); + dataManIO.SetEngine("ssc"); + dataManIO.SetParameters(engineParams); + adios2::Engine dataManReader = dataManIO.Open(name, adios2::Mode::Read); + + size_t datasize = std::accumulate(count.begin(), count.end(), 1, + std::multiplies()); + std::vector myChars(datasize); + std::vector myUChars(datasize); + std::vector myShorts(datasize); + std::vector myUShorts(datasize); + std::vector myInts(datasize); + std::vector myUInts(datasize); + std::vector myFloats(datasize); + std::vector myDoubles(datasize); + std::vector> myComplexes(datasize); + std::vector> myDComplexes(datasize); + + while (true) + { + adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5); + if (status == adios2::StepStatus::OK) + { + auto scalarInt = dataManIO.InquireVariable("scalarInt"); + auto blocksInfo = dataManReader.BlocksInfo( + scalarInt, dataManReader.CurrentStep()); + + for (const auto &bi : blocksInfo) + { + ASSERT_EQ(bi.IsValue, true); + ASSERT_EQ(bi.Value, dataManReader.CurrentStep()); + ASSERT_EQ(scalarInt.Min(), dataManReader.CurrentStep()); + ASSERT_EQ(scalarInt.Max(), dataManReader.CurrentStep()); + } + + const auto &vars = dataManIO.AvailableVariables(); + ASSERT_EQ(vars.size(), 12); + size_t currentStep = dataManReader.CurrentStep(); + adios2::Variable bpChars = + dataManIO.InquireVariable("bpChars"); + adios2::Variable bpUChars = + dataManIO.InquireVariable("bpUChars"); + adios2::Variable bpShorts = + dataManIO.InquireVariable("bpShorts"); + adios2::Variable bpUShorts = + dataManIO.InquireVariable("bpUShorts"); + adios2::Variable bpInts = + dataManIO.InquireVariable("bpInts"); + adios2::Variable bpUInts = + dataManIO.InquireVariable("bpUInts"); + adios2::Variable bpFloats = + dataManIO.InquireVariable("bpFloats"); + adios2::Variable bpDoubles = + dataManIO.InquireVariable("bpDoubles"); + adios2::Variable> bpComplexes = + dataManIO.InquireVariable>("bpComplexes"); + adios2::Variable> bpDComplexes = + dataManIO.InquireVariable>("bpDComplexes"); + adios2::Variable stringVar = + dataManIO.InquireVariable("stringVar"); + + bpChars.SetSelection({start, count}); + bpUChars.SetSelection({start, count}); + bpShorts.SetSelection({start, count}); + bpUShorts.SetSelection({start, count}); + bpInts.SetSelection({start, count}); + bpUInts.SetSelection({start, count}); + bpFloats.SetSelection({start, count}); + bpDoubles.SetSelection({start, count}); + bpComplexes.SetSelection({start, count}); + bpDComplexes.SetSelection({start, count}); + + dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync); + dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync); + dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync); + dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync); + dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync); + dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync); + dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync); + dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync); + dataManReader.Get(bpComplexes, myComplexes.data(), + adios2::Mode::Sync); + dataManReader.Get(bpDComplexes, myDComplexes.data(), + adios2::Mode::Sync); + std::string s; + dataManReader.Get(stringVar, s); + ASSERT_EQ(s, "sample string sample string sample string"); + ASSERT_EQ(stringVar.Min(), + "sample string sample string sample string"); + ASSERT_EQ(stringVar.Max(), + "sample string sample string sample string"); + + int i; + dataManReader.Get(scalarInt, &i); + ASSERT_EQ(i, currentStep); + + VerifyData(myChars.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myUChars.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myShorts.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myUShorts.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myInts.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myUInts.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myFloats.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myDoubles.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myComplexes.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myDComplexes.data(), currentStep, start, count, shape, + mpiRank); + dataManReader.EndStep(); + } + else if (status == adios2::StepStatus::EndOfStream) + { + std::cout << "[Rank " + std::to_string(mpiRank) + + "] SscTest reader end of stream!" + << std::endl; + break; + } + } + auto attInt = dataManIO.InquireAttribute("AttInt"); + std::cout << "[Rank " + std::to_string(mpiRank) + "] Attribute received " + << attInt.Data()[0] << ", expected 110" << std::endl; + ASSERT_EQ(110, attInt.Data()[0]); + ASSERT_NE(111, attInt.Data()[0]); + dataManReader.Close(); +} + +TEST_F(SscEngineTest, TestSscOnlyOneStep) +{ + std::string filename = "TestSscOnlyOneStep"; + adios2::Params engineParams = {}; + + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + Dims shape = {10, (size_t)mpiSize * 2}; + Dims start = {2, (size_t)mpiRank * 2}; + Dims count = {5, 2}; + size_t steps = 1; + + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); +} + +int main(int argc, char **argv) +{ + MPI_Init(&argc, &argv); + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + ::testing::InitGoogleTest(&argc, argv); + int result = RUN_ALL_TESTS(); + + MPI_Finalize(); + return result; +} diff --git a/testing/adios2/engine/ssc/TestSscOnlyTwoSteps.cpp b/testing/adios2/engine/ssc/TestSscOnlyTwoSteps.cpp new file mode 100644 index 0000000000..0c010273c8 --- /dev/null +++ b/testing/adios2/engine/ssc/TestSscOnlyTwoSteps.cpp @@ -0,0 +1,283 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ + +#include "TestSscCommon.h" +#include +#include +#include +#include +#include + +using namespace adios2; +int mpiRank = 0; +int mpiSize = 1; +MPI_Comm mpiComm; + +class SscEngineTest : public ::testing::Test +{ +public: + SscEngineTest() = default; +}; + +void Writer(const Dims &shape, const Dims &start, const Dims &count, + const size_t steps, const adios2::Params &engineParams, + const std::string &name) +{ + size_t datasize = std::accumulate(count.begin(), count.end(), 1, + std::multiplies()); + adios2::ADIOS adios(mpiComm); + adios2::IO dataManIO = adios.DeclareIO("WAN"); + dataManIO.SetEngine("ssc"); + dataManIO.SetParameters(engineParams); + std::vector myChars(datasize); + std::vector myUChars(datasize); + std::vector myShorts(datasize); + std::vector myUShorts(datasize); + std::vector myInts(datasize); + std::vector myUInts(datasize); + std::vector myFloats(datasize); + std::vector myDoubles(datasize); + std::vector> myComplexes(datasize); + std::vector> myDComplexes(datasize); + auto bpChars = + dataManIO.DefineVariable("bpChars", shape, start, count); + auto bpUChars = dataManIO.DefineVariable("bpUChars", shape, + start, count); + auto bpShorts = + dataManIO.DefineVariable("bpShorts", shape, start, count); + auto bpUShorts = dataManIO.DefineVariable( + "bpUShorts", shape, start, count); + auto bpInts = dataManIO.DefineVariable("bpInts", shape, start, count); + auto bpUInts = + dataManIO.DefineVariable("bpUInts", shape, start, count); + auto bpFloats = + dataManIO.DefineVariable("bpFloats", shape, start, count); + auto bpDoubles = + dataManIO.DefineVariable("bpDoubles", shape, start, count); + auto bpComplexes = dataManIO.DefineVariable>( + "bpComplexes", shape, start, count); + auto bpDComplexes = dataManIO.DefineVariable>( + "bpDComplexes", shape, start, count); + auto scalarInt = dataManIO.DefineVariable("scalarInt"); + auto stringVar = dataManIO.DefineVariable("stringVar"); + dataManIO.DefineAttribute("AttInt", 110); + adios2::Engine dataManWriter = dataManIO.Open(name, adios2::Mode::Write); + for (int i = 0; i < steps; ++i) + { + dataManWriter.BeginStep(); + GenData(myChars, i, start, count, shape); + GenData(myUChars, i, start, count, shape); + GenData(myShorts, i, start, count, shape); + GenData(myUShorts, i, start, count, shape); + GenData(myInts, i, start, count, shape); + GenData(myUInts, i, start, count, shape); + GenData(myFloats, i, start, count, shape); + GenData(myDoubles, i, start, count, shape); + GenData(myComplexes, i, start, count, shape); + GenData(myDComplexes, i, start, count, shape); + dataManWriter.Put(bpChars, myChars.data(), adios2::Mode::Sync); + dataManWriter.Put(bpUChars, myUChars.data(), adios2::Mode::Sync); + dataManWriter.Put(bpShorts, myShorts.data(), adios2::Mode::Sync); + dataManWriter.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync); + dataManWriter.Put(bpInts, myInts.data(), adios2::Mode::Sync); + dataManWriter.Put(bpUInts, myUInts.data(), adios2::Mode::Sync); + dataManWriter.Put(bpFloats, myFloats.data(), adios2::Mode::Sync); + dataManWriter.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync); + dataManWriter.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync); + dataManWriter.Put(bpDComplexes, myDComplexes.data(), + adios2::Mode::Sync); + dataManWriter.Put(scalarInt, i); + std::string s = "sample string sample string sample string"; + dataManWriter.Put(stringVar, s); + dataManWriter.EndStep(); + } + dataManWriter.Close(); +} + +void Reader(const Dims &shape, const Dims &start, const Dims &count, + const size_t steps, const adios2::Params &engineParams, + const std::string &name) +{ + adios2::ADIOS adios(mpiComm); + adios2::IO dataManIO = adios.DeclareIO("Test"); + dataManIO.SetEngine("ssc"); + dataManIO.SetParameters(engineParams); + adios2::Engine dataManReader = dataManIO.Open(name, adios2::Mode::Read); + + size_t datasize = std::accumulate(count.begin(), count.end(), 1, + std::multiplies()); + std::vector myChars(datasize); + std::vector myUChars(datasize); + std::vector myShorts(datasize); + std::vector myUShorts(datasize); + std::vector myInts(datasize); + std::vector myUInts(datasize); + std::vector myFloats(datasize); + std::vector myDoubles(datasize); + std::vector> myComplexes(datasize); + std::vector> myDComplexes(datasize); + + while (true) + { + adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5); + if (status == adios2::StepStatus::OK) + { + auto scalarInt = dataManIO.InquireVariable("scalarInt"); + auto blocksInfo = dataManReader.BlocksInfo( + scalarInt, dataManReader.CurrentStep()); + + for (const auto &bi : blocksInfo) + { + ASSERT_EQ(bi.IsValue, true); + ASSERT_EQ(bi.Value, dataManReader.CurrentStep()); + ASSERT_EQ(scalarInt.Min(), dataManReader.CurrentStep()); + ASSERT_EQ(scalarInt.Max(), dataManReader.CurrentStep()); + } + + const auto &vars = dataManIO.AvailableVariables(); + ASSERT_EQ(vars.size(), 12); + size_t currentStep = dataManReader.CurrentStep(); + adios2::Variable bpChars = + dataManIO.InquireVariable("bpChars"); + adios2::Variable bpUChars = + dataManIO.InquireVariable("bpUChars"); + adios2::Variable bpShorts = + dataManIO.InquireVariable("bpShorts"); + adios2::Variable bpUShorts = + dataManIO.InquireVariable("bpUShorts"); + adios2::Variable bpInts = + dataManIO.InquireVariable("bpInts"); + adios2::Variable bpUInts = + dataManIO.InquireVariable("bpUInts"); + adios2::Variable bpFloats = + dataManIO.InquireVariable("bpFloats"); + adios2::Variable bpDoubles = + dataManIO.InquireVariable("bpDoubles"); + adios2::Variable> bpComplexes = + dataManIO.InquireVariable>("bpComplexes"); + adios2::Variable> bpDComplexes = + dataManIO.InquireVariable>("bpDComplexes"); + adios2::Variable stringVar = + dataManIO.InquireVariable("stringVar"); + + bpChars.SetSelection({start, count}); + bpUChars.SetSelection({start, count}); + bpShorts.SetSelection({start, count}); + bpUShorts.SetSelection({start, count}); + bpInts.SetSelection({start, count}); + bpUInts.SetSelection({start, count}); + bpFloats.SetSelection({start, count}); + bpDoubles.SetSelection({start, count}); + bpComplexes.SetSelection({start, count}); + bpDComplexes.SetSelection({start, count}); + + dataManReader.Get(bpChars, myChars.data(), adios2::Mode::Sync); + dataManReader.Get(bpUChars, myUChars.data(), adios2::Mode::Sync); + dataManReader.Get(bpShorts, myShorts.data(), adios2::Mode::Sync); + dataManReader.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync); + dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync); + dataManReader.Get(bpUInts, myUInts.data(), adios2::Mode::Sync); + dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync); + dataManReader.Get(bpDoubles, myDoubles.data(), adios2::Mode::Sync); + dataManReader.Get(bpComplexes, myComplexes.data(), + adios2::Mode::Sync); + dataManReader.Get(bpDComplexes, myDComplexes.data(), + adios2::Mode::Sync); + std::string s; + dataManReader.Get(stringVar, s); + ASSERT_EQ(s, "sample string sample string sample string"); + ASSERT_EQ(stringVar.Min(), + "sample string sample string sample string"); + ASSERT_EQ(stringVar.Max(), + "sample string sample string sample string"); + + int i; + dataManReader.Get(scalarInt, &i); + ASSERT_EQ(i, currentStep); + + VerifyData(myChars.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myUChars.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myShorts.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myUShorts.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myInts.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myUInts.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myFloats.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myDoubles.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myComplexes.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myDComplexes.data(), currentStep, start, count, shape, + mpiRank); + dataManReader.EndStep(); + } + else if (status == adios2::StepStatus::EndOfStream) + { + std::cout << "[Rank " + std::to_string(mpiRank) + + "] SscTest reader end of stream!" + << std::endl; + break; + } + } + auto attInt = dataManIO.InquireAttribute("AttInt"); + std::cout << "[Rank " + std::to_string(mpiRank) + "] Attribute received " + << attInt.Data()[0] << ", expected 110" << std::endl; + ASSERT_EQ(110, attInt.Data()[0]); + ASSERT_NE(111, attInt.Data()[0]); + dataManReader.Close(); +} + +TEST_F(SscEngineTest, TestSscOnlyTwoSteps) +{ + std::string filename = "TestSscOnlyTwoSteps"; + adios2::Params engineParams = {}; + + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + Dims shape = {10, (size_t)mpiSize * 2}; + Dims start = {2, (size_t)mpiRank * 2}; + Dims count = {5, 2}; + size_t steps = 2; + + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); +} + +int main(int argc, char **argv) +{ + MPI_Init(&argc, &argv); + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + ::testing::InitGoogleTest(&argc, argv); + int result = RUN_ALL_TESTS(); + + MPI_Finalize(); + return result; +}