From e2f7f7ea6ce2b2a05000dda097d3de2c98f622af Mon Sep 17 00:00:00 2001 From: William F Godoy Date: Fri, 2 Feb 2018 14:55:11 -0500 Subject: [PATCH] Added tests for 2 threads --- testing/adios2/engine/bp/CMakeLists.txt | 5 + .../bp/TestBPWriteReadAsStreamADIOS2.cpp | 10 +- .../TestBPWriteReadAsStreamADIOS2_Threads.cpp | 1550 +++++++++++++++++ 3 files changed, 1559 insertions(+), 6 deletions(-) create mode 100644 testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2_Threads.cpp diff --git a/testing/adios2/engine/bp/CMakeLists.txt b/testing/adios2/engine/bp/CMakeLists.txt index 9b275262ce..3a593469f1 100644 --- a/testing/adios2/engine/bp/CMakeLists.txt +++ b/testing/adios2/engine/bp/CMakeLists.txt @@ -9,6 +9,9 @@ target_link_libraries(TestBPWriteReadADIOS2 adios2 gtest gtest_main) add_executable(TestBPWriteReadAsStreamADIOS2 TestBPWriteReadAsStreamADIOS2.cpp) target_link_libraries(TestBPWriteReadAsStreamADIOS2 adios2 gtest gtest_main) +add_executable(TestBPWriteReadAsStreamADIOS2_Threads TestBPWriteReadAsStreamADIOS2_Threads.cpp) +target_link_libraries(TestBPWriteReadAsStreamADIOS2_Threads adios2 gtest gtest_main) + add_executable(TestBPWriteReadAttributesADIOS2 TestBPWriteReadAttributesADIOS2.cpp) target_link_libraries(TestBPWriteReadAttributesADIOS2 adios2 gtest gtest_main) @@ -16,6 +19,7 @@ target_link_libraries(TestBPWriteReadAttributesADIOS2 adios2 gtest gtest_main) if(ADIOS2_HAVE_MPI) target_link_libraries(TestBPWriteReadADIOS2 MPI::MPI_C) target_link_libraries(TestBPWriteReadAsStreamADIOS2 MPI::MPI_C) + target_link_libraries(TestBPWriteReadAsStreamADIOS2_Threads MPI::MPI_C) target_link_libraries(TestBPWriteReadAttributesADIOS2 MPI::MPI_C) set(extra_test_args EXEC_WRAPPER ${MPIEXEC_COMMAND}) @@ -23,6 +27,7 @@ endif() gtest_add_tests(TARGET TestBPWriteReadADIOS2 ${extra_test_args}) gtest_add_tests(TARGET TestBPWriteReadAsStreamADIOS2 ${extra_test_args}) +gtest_add_tests(TARGET TestBPWriteReadAsStreamADIOS2_Threads ${extra_test_args}) gtest_add_tests(TARGET TestBPWriteReadAttributesADIOS2 ${extra_test_args}) if (ADIOS2_HAVE_ADIOS1) diff --git a/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2.cpp b/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2.cpp index 7976bd15ff..e8f5cf1a21 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2.cpp @@ -101,7 +101,7 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ADIOS2BPWriteRead1D8) // Create the BP Engine io.SetEngine("BPFile"); - io.SetParameter("Threads", "2"); + io.AddTransport("file"); adios2::Engine &bpWriter = io.Open(fname, adios2::Mode::Write); @@ -346,7 +346,6 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ADIOS2BPWriteRead2D2x4) // Create the BP Engine io.SetEngine("BPFile"); - io.SetParameter("Threads", "2"); io.AddTransport("file"); adios2::Engine &bpWriter = io.Open(fname, adios2::Mode::Write); @@ -600,7 +599,7 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ADIOS2BPWriteRead2D4x2) // Create the BP Engine io.SetEngine("BPFile"); - io.SetParameter("Threads", "2"); + io.AddTransport("file"); adios2::Engine &bpWriter = io.Open(fname, adios2::Mode::Write); @@ -850,7 +849,7 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ADIOS2BPWriteRead1D8MissingPerformGets) // Create the BP Engine io.SetEngine("BPFile"); - io.SetParameter("Threads", "2"); + io.AddTransport("file"); // QUESTION: It seems that BPFilterWriter cannot overwrite existing @@ -1100,7 +1099,6 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ADIOS2BPWriteRead2D2x4MissingPerformGets) // Create the BP Engine io.SetEngine("BPFile"); - io.SetParameter("Threads", "2"); io.AddTransport("file"); adios2::Engine &bpWriter = io.Open(fname, adios2::Mode::Write); @@ -1354,7 +1352,7 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ADIOS2BPWriteRead2D4x2MissingPerformGets) // Create the BP Engine io.SetEngine("BPFile"); - io.SetParameter("Threads", "2"); + io.AddTransport("file"); adios2::Engine &bpWriter = io.Open(fname, adios2::Mode::Write); diff --git a/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2_Threads.cpp b/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2_Threads.cpp new file mode 100644 index 0000000000..928151134a --- /dev/null +++ b/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2_Threads.cpp @@ -0,0 +1,1550 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ +#include +#include + +#include +#include + +#include + +#include + +#include "../SmallTestData.h" + +class BPWriteReadAsStreamTestADIOS2_Threads : public ::testing::Test +{ +public: + BPWriteReadAsStreamTestADIOS2_Threads() = default; + + SmallTestData m_TestData; + SmallTestData m_OriginalData; +}; + +//****************************************************************************** +// 1D 1x8 test data +//****************************************************************************** + +// ADIOS2 BP write, native ADIOS1 read +TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, ADIOS2BPWriteRead1D8) +{ + // Each process would write a 1x8 array and all processes would + // form a mpiSize * Nx 1D array + const std::string fname("ADIOS2BPWriteReadAsStream1D8.bp"); + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const size_t Nx = 8; + + // Number of steps + const size_t NSteps = 3; + +#ifdef ADIOS2_HAVE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + +// Write test data using BP + +#ifdef ADIOS2_HAVE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON); +#else + adios2::ADIOS adios(true); +#endif + { + adios2::IO &io = adios.DeclareIO("TestIO"); + + // Declare 1D variables (NumOfProcesses * Nx) + // The local process' part (start, count) can be defined now or later + // before Write(). + { + const adios2::Dims shape{static_cast(Nx * mpiSize)}; + const adios2::Dims start{static_cast(Nx * mpiRank)}; + const adios2::Dims count{Nx}; + + io.DefineVariable("i8", shape, start, count, + adios2::ConstantDims, + m_TestData.I8.data()); + io.DefineVariable("i16", shape, start, count, + adios2::ConstantDims, + m_TestData.I16.data()); + io.DefineVariable("i32", shape, start, count, + adios2::ConstantDims, + m_TestData.I32.data()); + io.DefineVariable("i64", shape, start, count, + adios2::ConstantDims, + m_TestData.I64.data()); + + io.DefineVariable("u8", shape, start, count, + adios2::ConstantDims, + m_TestData.U8.data()); + + io.DefineVariable("u16", shape, start, count, + adios2::ConstantDims, + m_TestData.U16.data()); + io.DefineVariable("u32", shape, start, count, + adios2::ConstantDims, + m_TestData.U32.data()); + io.DefineVariable("u64", shape, start, count, + adios2::ConstantDims, + m_TestData.U64.data()); + + io.DefineVariable("r32", shape, start, count, + adios2::ConstantDims, + m_TestData.R32.data()); + io.DefineVariable("r64", shape, start, count, + adios2::ConstantDims, + m_TestData.R64.data()); + } + + // Create the BP Engine + io.SetEngine("BPFile"); + io.SetParameter("Threads", "2"); + io.AddTransport("file"); + + adios2::Engine &bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + UpdateSmallTestData(m_TestData, static_cast(step), mpiRank, + mpiSize); + EXPECT_EQ(bpWriter.CurrentStep(), step); + bpWriter.WriteStep(); + } + + bpWriter.Close(); + } + + { + adios2::IO &io = adios.DeclareIO("ReadIO"); + io.SetParameter("Threads", "2"); + adios2::Engine &bpReader = io.Open(fname, adios2::Mode::Read); + + auto var_i8 = io.InquireVariable("i8"); + ASSERT_NE(var_i8, nullptr); + ASSERT_EQ(var_i8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i8->m_Shape[0], mpiSize * Nx); + + auto var_i16 = io.InquireVariable("i16"); + ASSERT_NE(var_i16, nullptr); + ASSERT_EQ(var_i16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i16->m_Shape[0], mpiSize * Nx); + + auto var_i32 = io.InquireVariable("i32"); + ASSERT_NE(var_i32, nullptr); + ASSERT_EQ(var_i32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i32->m_Shape[0], mpiSize * Nx); + + auto var_i64 = io.InquireVariable("i64"); + ASSERT_NE(var_i64, nullptr); + ASSERT_EQ(var_i64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i64->m_Shape[0], mpiSize * Nx); + + auto var_u8 = io.InquireVariable("u8"); + ASSERT_NE(var_u8, nullptr); + ASSERT_EQ(var_u8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u8->m_Shape[0], mpiSize * Nx); + + auto var_u16 = io.InquireVariable("u16"); + ASSERT_NE(var_u16, nullptr); + ASSERT_EQ(var_u16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u16->m_Shape[0], mpiSize * Nx); + + auto var_u32 = io.InquireVariable("u32"); + ASSERT_NE(var_u32, nullptr); + ASSERT_EQ(var_u32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u32->m_Shape[0], mpiSize * Nx); + + auto var_u64 = io.InquireVariable("u64"); + ASSERT_NE(var_u64, nullptr); + ASSERT_EQ(var_u64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u64->m_Shape[0], mpiSize * Nx); + + auto var_r32 = io.InquireVariable("r32"); + ASSERT_NE(var_r32, nullptr); + ASSERT_EQ(var_r32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r32->m_Shape[0], mpiSize * Nx); + + auto var_r64 = io.InquireVariable("r64"); + ASSERT_NE(var_r64, nullptr); + ASSERT_EQ(var_r64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r64->m_Shape[0], mpiSize * Nx); + + std::string IString; + std::array I8; + std::array I16; + std::array I32; + std::array I64; + std::array U8; + std::array U16; + std::array U32; + std::array U64; + std::array R32; + std::array R64; + + const adios2::Dims start{mpiRank * Nx}; + const adios2::Dims count{Nx}; + + const adios2::Box sel(start, count); + + var_i8->SetSelection(sel); + var_i16->SetSelection(sel); + var_i32->SetSelection(sel); + var_i64->SetSelection(sel); + + var_u8->SetSelection(sel); + var_u16->SetSelection(sel); + var_u32->SetSelection(sel); + var_u64->SetSelection(sel); + + var_r32->SetSelection(sel); + var_r64->SetSelection(sel); + + unsigned int t = 0; + + while (bpReader.BeginStep() == adios2::StepStatus::OK) + { + const size_t currentStep = bpReader.CurrentStep(); + EXPECT_EQ(currentStep, static_cast(t)); + + bpReader.GetDeferred(*var_i8, I8.data()); + bpReader.GetDeferred(*var_i16, I16.data()); + bpReader.GetDeferred(*var_i32, I32.data()); + bpReader.GetDeferred(*var_i64, I64.data()); + + bpReader.GetDeferred(*var_u8, U8.data()); + bpReader.GetDeferred(*var_u16, U16.data()); + bpReader.GetDeferred(*var_u32, U32.data()); + bpReader.GetDeferred(*var_u64, U64.data()); + + bpReader.GetDeferred(*var_r32, R32.data()); + bpReader.GetDeferred(*var_r64, R64.data()); + + bpReader.PerformGets(); + + bpReader.EndStep(); + + UpdateSmallTestData(m_OriginalData, static_cast(t), mpiRank, + mpiSize); + + for (size_t i = 0; i < Nx; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + EXPECT_EQ(I8[i], m_OriginalData.I8[i]) << msg; + EXPECT_EQ(I16[i], m_OriginalData.I16[i]) << msg; + EXPECT_EQ(I32[i], m_OriginalData.I32[i]) << msg; + EXPECT_EQ(I64[i], m_OriginalData.I64[i]) << msg; + EXPECT_EQ(U8[i], m_OriginalData.U8[i]) << msg; + EXPECT_EQ(U16[i], m_OriginalData.U16[i]) << msg; + EXPECT_EQ(U32[i], m_OriginalData.U32[i]) << msg; + EXPECT_EQ(U64[i], m_OriginalData.U64[i]) << msg; + EXPECT_EQ(R32[i], m_OriginalData.R32[i]) << msg; + EXPECT_EQ(R64[i], m_OriginalData.R64[i]) << msg; + } + ++t; + } + + EXPECT_EQ(t, NSteps); + + bpReader.Close(); + } +} + +//****************************************************************************** +// 2D 2x4 test data +//****************************************************************************** + +// ADIOS2 BP write, native ADIOS1 read +TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, ADIOS2BPWriteRead2D2x4) +{ + // Each process would write a 2x4 array and all processes would + // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here + const std::string fname("ADIOS2BPWriteReadAsStream2D2x4Test.bp"); + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const std::size_t Nx = 4; + + // Number of rows + const std::size_t Ny = 2; + + // Number of steps + const std::size_t NSteps = 3; + +#ifdef ADIOS2_HAVE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + +// Write test data using ADIOS2 + +#ifdef ADIOS2_HAVE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON); +#else + adios2::ADIOS adios(true); +#endif + { + adios2::IO &io = adios.DeclareIO("TestIO"); + io.SetParameter("Threads", "2"); + // Declare 2D variables (Ny * (NumOfProcesses * Nx)) + // The local process' part (start, count) can be defined now or later + // before Write(). + { + const adios2::Dims shape{Ny, static_cast(Nx * mpiSize)}; + const adios2::Dims start{0, static_cast(mpiRank * Nx)}; + const adios2::Dims count{Ny, Nx}; + + io.DefineVariable("i8", shape, start, count, + adios2::ConstantDims, + m_TestData.I8.data()); + io.DefineVariable("i16", shape, start, count, + adios2::ConstantDims, + m_TestData.I16.data()); + io.DefineVariable("i32", shape, start, count, + adios2::ConstantDims, + m_TestData.I32.data()); + io.DefineVariable("i64", shape, start, count, + adios2::ConstantDims, + m_TestData.I64.data()); + + io.DefineVariable("u8", shape, start, count, + adios2::ConstantDims, + m_TestData.U8.data()); + + io.DefineVariable("u16", shape, start, count, + adios2::ConstantDims, + m_TestData.U16.data()); + io.DefineVariable("u32", shape, start, count, + adios2::ConstantDims, + m_TestData.U32.data()); + io.DefineVariable("u64", shape, start, count, + adios2::ConstantDims, + m_TestData.U64.data()); + + io.DefineVariable("r32", shape, start, count, + adios2::ConstantDims, + m_TestData.R32.data()); + io.DefineVariable("r64", shape, start, count, + adios2::ConstantDims, + m_TestData.R64.data()); + } + + // Create the BP Engine + io.SetEngine("BPFile"); + io.SetParameter("Threads", "2"); + io.AddTransport("file"); + + adios2::Engine &bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + UpdateSmallTestData(m_TestData, static_cast(step), mpiRank, + mpiSize); + EXPECT_EQ(bpWriter.CurrentStep(), step); + bpWriter.WriteStep(); + } + + bpWriter.Close(); + } + + { + adios2::IO &io = adios.DeclareIO("ReadIO"); + io.SetParameter("Threads", "2"); + adios2::Engine &bpReader = io.Open(fname, adios2::Mode::Read); + + auto var_i8 = io.InquireVariable("i8"); + ASSERT_NE(var_i8, nullptr); + ASSERT_EQ(var_i8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i8->m_Shape[0], Ny); + ASSERT_EQ(var_i8->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i16 = io.InquireVariable("i16"); + ASSERT_NE(var_i16, nullptr); + ASSERT_EQ(var_i16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i16->m_Shape[0], Ny); + ASSERT_EQ(var_i16->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i32 = io.InquireVariable("i32"); + ASSERT_NE(var_i32, nullptr); + ASSERT_EQ(var_i32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i32->m_Shape[0], Ny); + ASSERT_EQ(var_i32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i64 = io.InquireVariable("i64"); + ASSERT_NE(var_i64, nullptr); + ASSERT_EQ(var_i64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i64->m_Shape[0], Ny); + ASSERT_EQ(var_i64->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u8 = io.InquireVariable("u8"); + ASSERT_NE(var_u8, nullptr); + ASSERT_EQ(var_u8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u8->m_Shape[0], Ny); + ASSERT_EQ(var_u8->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u16 = io.InquireVariable("u16"); + ASSERT_NE(var_u16, nullptr); + ASSERT_EQ(var_u16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u16->m_Shape[0], Ny); + ASSERT_EQ(var_u16->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u32 = io.InquireVariable("u32"); + ASSERT_NE(var_u32, nullptr); + ASSERT_EQ(var_u32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u32->m_Shape[0], Ny); + ASSERT_EQ(var_u32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u64 = io.InquireVariable("u64"); + ASSERT_NE(var_u64, nullptr); + ASSERT_EQ(var_u64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u64->m_Shape[0], Ny); + ASSERT_EQ(var_u64->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_r32 = io.InquireVariable("r32"); + ASSERT_NE(var_r32, nullptr); + ASSERT_EQ(var_r32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r32->m_Shape[0], Ny); + ASSERT_EQ(var_r32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_r64 = io.InquireVariable("r64"); + ASSERT_NE(var_r64, nullptr); + ASSERT_EQ(var_r64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r64->m_Shape[0], Ny); + ASSERT_EQ(var_r64->m_Shape[1], static_cast(mpiSize * Nx)); + + std::array I8; + std::array I16; + std::array I32; + std::array I64; + std::array U8; + std::array U16; + std::array U32; + std::array U64; + std::array R32; + std::array R64; + + const adios2::Dims start{0, static_cast(mpiRank * Nx)}; + const adios2::Dims count{Ny, Nx}; + + const adios2::Box sel(start, count); + + var_i8->SetSelection(sel); + var_i16->SetSelection(sel); + var_i32->SetSelection(sel); + var_i64->SetSelection(sel); + + var_u8->SetSelection(sel); + var_u16->SetSelection(sel); + var_u32->SetSelection(sel); + var_u64->SetSelection(sel); + + var_r32->SetSelection(sel); + var_r64->SetSelection(sel); + + unsigned int t = 0; + + while (bpReader.BeginStep() == adios2::StepStatus::OK) + { + const size_t currentStep = bpReader.CurrentStep(); + EXPECT_EQ(currentStep, static_cast(t)); + + bpReader.GetDeferred(*var_i8, I8.data()); + bpReader.GetDeferred(*var_i16, I16.data()); + bpReader.GetDeferred(*var_i32, I32.data()); + bpReader.GetDeferred(*var_i64, I64.data()); + + bpReader.GetDeferred(*var_u8, U8.data()); + bpReader.GetDeferred(*var_u16, U16.data()); + bpReader.GetDeferred(*var_u32, U32.data()); + bpReader.GetDeferred(*var_u64, U64.data()); + + bpReader.GetDeferred(*var_r32, R32.data()); + bpReader.GetDeferred(*var_r64, R64.data()); + + bpReader.PerformGets(); + bpReader.EndStep(); + + // Generate test data for each rank uniquely + UpdateSmallTestData(m_OriginalData, static_cast(t), mpiRank, + mpiSize); + + for (size_t i = 0; i < Nx * Ny; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + EXPECT_EQ(I8[i], m_OriginalData.I8[i]) << msg; + EXPECT_EQ(I16[i], m_OriginalData.I16[i]) << msg; + EXPECT_EQ(I32[i], m_OriginalData.I32[i]) << msg; + EXPECT_EQ(I64[i], m_OriginalData.I64[i]) << msg; + EXPECT_EQ(U8[i], m_OriginalData.U8[i]) << msg; + EXPECT_EQ(U16[i], m_OriginalData.U16[i]) << msg; + EXPECT_EQ(U32[i], m_OriginalData.U32[i]) << msg; + EXPECT_EQ(U64[i], m_OriginalData.U64[i]) << msg; + EXPECT_EQ(R32[i], m_OriginalData.R32[i]) << msg; + EXPECT_EQ(R64[i], m_OriginalData.R64[i]) << msg; + } + ++t; + } + EXPECT_EQ(t, NSteps); + + bpReader.Close(); + } +} + +//****************************************************************************** +// 2D 4x2 test data +//****************************************************************************** + +TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, ADIOS2BPWriteRead2D4x2) +{ + // Each process would write a 4x2 array and all processes would + // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here + const std::string fname("ADIOS2BPWriteReadAsStream2D4x2Test.bp"); + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const std::size_t Nx = 2; + // Number of cols + const std::size_t Ny = 4; + + // Number of steps + const std::size_t NSteps = 3; + +#ifdef ADIOS2_HAVE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + +// Write test data using ADIOS2 + +#ifdef ADIOS2_HAVE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON); +#else + adios2::ADIOS adios(true); +#endif + { + adios2::IO &io = adios.DeclareIO("TestIO"); + io.SetParameter("Threads", "2"); + // Declare 2D variables (4 * (NumberOfProcess * Nx)) + // The local process' part (start, count) can be defined now or later + // before Write(). + { + adios2::Dims shape{static_cast(Ny), + static_cast(mpiSize * Nx)}; + adios2::Dims start{static_cast(0), + static_cast(mpiRank * Nx)}; + adios2::Dims count{static_cast(Ny), + static_cast(Nx)}; + + io.DefineVariable("i8", shape, start, count, + adios2::ConstantDims, + m_TestData.I8.data()); + io.DefineVariable("i16", shape, start, count, + adios2::ConstantDims, + m_TestData.I16.data()); + io.DefineVariable("i32", shape, start, count, + adios2::ConstantDims, + m_TestData.I32.data()); + io.DefineVariable("i64", shape, start, count, + adios2::ConstantDims, + m_TestData.I64.data()); + + io.DefineVariable("u8", shape, start, count, + adios2::ConstantDims, + m_TestData.U8.data()); + + io.DefineVariable("u16", shape, start, count, + adios2::ConstantDims, + m_TestData.U16.data()); + io.DefineVariable("u32", shape, start, count, + adios2::ConstantDims, + m_TestData.U32.data()); + io.DefineVariable("u64", shape, start, count, + adios2::ConstantDims, + m_TestData.U64.data()); + + io.DefineVariable("r32", shape, start, count, + adios2::ConstantDims, + m_TestData.R32.data()); + io.DefineVariable("r64", shape, start, count, + adios2::ConstantDims, + m_TestData.R64.data()); + } + + // Create the BP Engine + io.SetEngine("BPFile"); + io.SetParameter("Threads", "2"); + io.AddTransport("file"); + + adios2::Engine &bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + // Generate test data for each process uniquely + UpdateSmallTestData(m_TestData, static_cast(step), mpiRank, + mpiSize); + EXPECT_EQ(bpWriter.CurrentStep(), step); + bpWriter.WriteStep(); + } + + bpWriter.Close(); + } + + { + adios2::IO &io = adios.DeclareIO("ReadIO"); + io.SetParameter("Threads", "2"); + adios2::Engine &bpReader = io.Open(fname, adios2::Mode::Read); + + auto var_i8 = io.InquireVariable("i8"); + ASSERT_NE(var_i8, nullptr); + ASSERT_EQ(var_i8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i8->m_Shape[0], Ny); + ASSERT_EQ(var_i8->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i16 = io.InquireVariable("i16"); + ASSERT_NE(var_i16, nullptr); + ASSERT_EQ(var_i16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i16->m_Shape[0], Ny); + ASSERT_EQ(var_i16->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i32 = io.InquireVariable("i32"); + ASSERT_NE(var_i32, nullptr); + ASSERT_EQ(var_i32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i32->m_Shape[0], Ny); + ASSERT_EQ(var_i32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i64 = io.InquireVariable("i64"); + ASSERT_NE(var_i64, nullptr); + ASSERT_EQ(var_i64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i64->m_Shape[0], Ny); + ASSERT_EQ(var_i64->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u8 = io.InquireVariable("u8"); + ASSERT_NE(var_u8, nullptr); + ASSERT_EQ(var_u8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u8->m_Shape[0], Ny); + ASSERT_EQ(var_u8->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u16 = io.InquireVariable("u16"); + ASSERT_NE(var_u16, nullptr); + ASSERT_EQ(var_u16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u16->m_Shape[0], Ny); + ASSERT_EQ(var_u16->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u32 = io.InquireVariable("u32"); + ASSERT_NE(var_u32, nullptr); + ASSERT_EQ(var_u32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u32->m_Shape[0], Ny); + ASSERT_EQ(var_u32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u64 = io.InquireVariable("u64"); + ASSERT_NE(var_u64, nullptr); + ASSERT_EQ(var_u64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u64->m_Shape[0], Ny); + ASSERT_EQ(var_u64->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_r32 = io.InquireVariable("r32"); + ASSERT_NE(var_r32, nullptr); + ASSERT_EQ(var_r32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r32->m_Shape[0], Ny); + ASSERT_EQ(var_r32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_r64 = io.InquireVariable("r64"); + ASSERT_NE(var_r64, nullptr); + ASSERT_EQ(var_r64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r64->m_Shape[0], Ny); + ASSERT_EQ(var_r64->m_Shape[1], static_cast(mpiSize * Nx)); + + // If the size of the array is smaller than the data + // the result is weird... double and uint64_t would get + // completely garbage data + std::array I8; + std::array I16; + std::array I32; + std::array I64; + std::array U8; + std::array U16; + std::array U32; + std::array U64; + std::array R32; + std::array R64; + + const adios2::Dims start{0, static_cast(mpiRank * Nx)}; + const adios2::Dims count{Ny, Nx}; + + const adios2::Box sel(start, count); + + var_i8->SetSelection(sel); + var_i16->SetSelection(sel); + var_i32->SetSelection(sel); + var_i64->SetSelection(sel); + + var_u8->SetSelection(sel); + var_u16->SetSelection(sel); + var_u32->SetSelection(sel); + var_u64->SetSelection(sel); + + var_r32->SetSelection(sel); + var_r64->SetSelection(sel); + + unsigned int t = 0; + + while (bpReader.BeginStep() == adios2::StepStatus::OK) + { + const size_t currentStep = bpReader.CurrentStep(); + EXPECT_EQ(currentStep, static_cast(t)); + + bpReader.GetDeferred(*var_i8, I8.data()); + bpReader.GetDeferred(*var_i16, I16.data()); + bpReader.GetDeferred(*var_i32, I32.data()); + bpReader.GetDeferred(*var_i64, I64.data()); + + bpReader.GetDeferred(*var_u8, U8.data()); + bpReader.GetDeferred(*var_u16, U16.data()); + bpReader.GetDeferred(*var_u32, U32.data()); + bpReader.GetDeferred(*var_u64, U64.data()); + + bpReader.GetDeferred(*var_r32, R32.data()); + bpReader.GetDeferred(*var_r64, R64.data()); + + bpReader.PerformGets(); + + bpReader.EndStep(); + + // Generate test data for each rank uniquely + UpdateSmallTestData(m_OriginalData, static_cast(t), mpiRank, + mpiSize); + + for (size_t i = 0; i < Nx * Ny; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + EXPECT_EQ(I8[i], m_OriginalData.I8[i]) << msg; + EXPECT_EQ(I16[i], m_OriginalData.I16[i]) << msg; + EXPECT_EQ(I32[i], m_OriginalData.I32[i]) << msg; + EXPECT_EQ(I64[i], m_OriginalData.I64[i]) << msg; + EXPECT_EQ(U8[i], m_OriginalData.U8[i]) << msg; + EXPECT_EQ(U16[i], m_OriginalData.U16[i]) << msg; + EXPECT_EQ(U32[i], m_OriginalData.U32[i]) << msg; + EXPECT_EQ(U64[i], m_OriginalData.U64[i]) << msg; + EXPECT_EQ(R32[i], m_OriginalData.R32[i]) << msg; + EXPECT_EQ(R64[i], m_OriginalData.R64[i]) << msg; + } + ++t; + } + EXPECT_EQ(t, NSteps); + bpReader.Close(); + } +} + +TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, + ADIOS2BPWriteRead1D8MissingPerformGets) +{ + // Each process would write a 1x8 array and all processes would + // form a mpiSize * Nx 1D array + const std::string fname( + "ADIOS2BPWriteReadAsStream1D8MissingPerformGetsTest.bp"); + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const size_t Nx = 8; + + // Number of steps + const size_t NSteps = 3; + +#ifdef ADIOS2_HAVE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + +// Write test data using BP + +#ifdef ADIOS2_HAVE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON); +#else + adios2::ADIOS adios(true); +#endif + { + adios2::IO &io = adios.DeclareIO("TestIO"); + // Declare 1D variables (NumOfProcesses * Nx) + // The local process' part (start, count) can be defined now or later + // before Write(). + { + const adios2::Dims shape{static_cast(Nx * mpiSize)}; + const adios2::Dims start{static_cast(Nx * mpiRank)}; + const adios2::Dims count{Nx}; + + io.DefineVariable("i8", shape, start, count, + adios2::ConstantDims, + m_TestData.I8.data()); + io.DefineVariable("i16", shape, start, count, + adios2::ConstantDims, + m_TestData.I16.data()); + io.DefineVariable("i32", shape, start, count, + adios2::ConstantDims, + m_TestData.I32.data()); + io.DefineVariable("i64", shape, start, count, + adios2::ConstantDims, + m_TestData.I64.data()); + + io.DefineVariable("u8", shape, start, count, + adios2::ConstantDims, + m_TestData.U8.data()); + + io.DefineVariable("u16", shape, start, count, + adios2::ConstantDims, + m_TestData.U16.data()); + io.DefineVariable("u32", shape, start, count, + adios2::ConstantDims, + m_TestData.U32.data()); + io.DefineVariable("u64", shape, start, count, + adios2::ConstantDims, + m_TestData.U64.data()); + + io.DefineVariable("r32", shape, start, count, + adios2::ConstantDims, + m_TestData.R32.data()); + io.DefineVariable("r64", shape, start, count, + adios2::ConstantDims, + m_TestData.R64.data()); + } + + // Create the BP Engine + io.SetEngine("BPFile"); + io.SetParameter("Threads", "2"); + io.AddTransport("file"); + + // QUESTION: It seems that BPFilterWriter cannot overwrite existing + // files + // Ex. if you tune Nx and NSteps, the test would fail. But if you clear + // the cache in + // ${adios2Build}/testing/adios2/engine/bp/ADIOS2BPWriteADIOS1Read1D8.bp.dir, + // then it works + adios2::Engine &bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + UpdateSmallTestData(m_TestData, static_cast(step), mpiRank, + mpiSize); + EXPECT_EQ(bpWriter.CurrentStep(), step); + bpWriter.WriteStep(); + } + + bpWriter.Close(); + } + + { + adios2::IO &io = adios.DeclareIO("ReadIO"); + io.SetParameter("Threads", "2"); + + adios2::Engine &bpReader = io.Open(fname, adios2::Mode::Read); + + auto var_i8 = io.InquireVariable("i8"); + ASSERT_NE(var_i8, nullptr); + ASSERT_EQ(var_i8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i8->m_Shape[0], mpiSize * Nx); + + auto var_i16 = io.InquireVariable("i16"); + ASSERT_NE(var_i16, nullptr); + ASSERT_EQ(var_i16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i16->m_Shape[0], mpiSize * Nx); + + auto var_i32 = io.InquireVariable("i32"); + ASSERT_NE(var_i32, nullptr); + ASSERT_EQ(var_i32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i32->m_Shape[0], mpiSize * Nx); + + auto var_i64 = io.InquireVariable("i64"); + ASSERT_NE(var_i64, nullptr); + ASSERT_EQ(var_i64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i64->m_Shape[0], mpiSize * Nx); + + auto var_u8 = io.InquireVariable("u8"); + ASSERT_NE(var_u8, nullptr); + ASSERT_EQ(var_u8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u8->m_Shape[0], mpiSize * Nx); + + auto var_u16 = io.InquireVariable("u16"); + ASSERT_NE(var_u16, nullptr); + ASSERT_EQ(var_u16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u16->m_Shape[0], mpiSize * Nx); + + auto var_u32 = io.InquireVariable("u32"); + ASSERT_NE(var_u32, nullptr); + ASSERT_EQ(var_u32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u32->m_Shape[0], mpiSize * Nx); + + auto var_u64 = io.InquireVariable("u64"); + ASSERT_NE(var_u64, nullptr); + ASSERT_EQ(var_u64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u64->m_Shape[0], mpiSize * Nx); + + auto var_r32 = io.InquireVariable("r32"); + ASSERT_NE(var_r32, nullptr); + ASSERT_EQ(var_r32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r32->m_Shape[0], mpiSize * Nx); + + auto var_r64 = io.InquireVariable("r64"); + ASSERT_NE(var_r64, nullptr); + ASSERT_EQ(var_r64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r64->m_Shape[0], mpiSize * Nx); + + std::string IString; + std::array I8; + std::array I16; + std::array I32; + std::array I64; + std::array U8; + std::array U16; + std::array U32; + std::array U64; + std::array R32; + std::array R64; + + const adios2::Dims start{mpiRank * Nx}; + const adios2::Dims count{Nx}; + + const adios2::Box sel(start, count); + + var_i8->SetSelection(sel); + var_i16->SetSelection(sel); + var_i32->SetSelection(sel); + var_i64->SetSelection(sel); + + var_u8->SetSelection(sel); + var_u16->SetSelection(sel); + var_u32->SetSelection(sel); + var_u64->SetSelection(sel); + + var_r32->SetSelection(sel); + var_r64->SetSelection(sel); + + unsigned int t = 0; + + while (bpReader.BeginStep() == adios2::StepStatus::OK) + { + const size_t currentStep = bpReader.CurrentStep(); + EXPECT_EQ(currentStep, static_cast(t)); + + bpReader.GetDeferred(*var_i8, I8.data()); + bpReader.GetDeferred(*var_i16, I16.data()); + bpReader.GetDeferred(*var_i32, I32.data()); + bpReader.GetDeferred(*var_i64, I64.data()); + + bpReader.GetDeferred(*var_u8, U8.data()); + bpReader.GetDeferred(*var_u16, U16.data()); + bpReader.GetDeferred(*var_u32, U32.data()); + bpReader.GetDeferred(*var_u64, U64.data()); + + bpReader.GetDeferred(*var_r32, R32.data()); + bpReader.GetDeferred(*var_r64, R64.data()); + + bpReader.EndStep(); + + UpdateSmallTestData(m_OriginalData, static_cast(t), mpiRank, + mpiSize); + + for (size_t i = 0; i < Nx; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + EXPECT_EQ(I8[i], m_OriginalData.I8[i]) << msg; + EXPECT_EQ(I16[i], m_OriginalData.I16[i]) << msg; + EXPECT_EQ(I32[i], m_OriginalData.I32[i]) << msg; + EXPECT_EQ(I64[i], m_OriginalData.I64[i]) << msg; + EXPECT_EQ(U8[i], m_OriginalData.U8[i]) << msg; + EXPECT_EQ(U16[i], m_OriginalData.U16[i]) << msg; + EXPECT_EQ(U32[i], m_OriginalData.U32[i]) << msg; + EXPECT_EQ(U64[i], m_OriginalData.U64[i]) << msg; + EXPECT_EQ(R32[i], m_OriginalData.R32[i]) << msg; + EXPECT_EQ(R64[i], m_OriginalData.R64[i]) << msg; + } + ++t; + } + + EXPECT_EQ(t, NSteps); + + bpReader.Close(); + } +} + +//****************************************************************************** +// 2D 2x4 test data +//****************************************************************************** + +// ADIOS2 BP write, native ADIOS1 read +TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, + ADIOS2BPWriteRead2D2x4MissingPerformGets) +{ + // Each process would write a 2x4 array and all processes would + // form a 2D 2 * (numberOfProcess*Nx) matrix where Nx is 4 here + const std::string fname( + "ADIOS2BPWriteReadAsStream2D2x4MissingPerformGetsTest.bp"); + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const std::size_t Nx = 4; + + // Number of rows + const std::size_t Ny = 2; + + // Number of steps + const std::size_t NSteps = 3; + +#ifdef ADIOS2_HAVE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + +// Write test data using ADIOS2 + +#ifdef ADIOS2_HAVE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON); +#else + adios2::ADIOS adios(true); +#endif + { + adios2::IO &io = adios.DeclareIO("TestIO"); + // Declare 2D variables (Ny * (NumOfProcesses * Nx)) + // The local process' part (start, count) can be defined now or later + // before Write(). + { + const adios2::Dims shape{Ny, static_cast(Nx * mpiSize)}; + const adios2::Dims start{0, static_cast(mpiRank * Nx)}; + const adios2::Dims count{Ny, Nx}; + + io.DefineVariable("i8", shape, start, count, + adios2::ConstantDims, + m_TestData.I8.data()); + io.DefineVariable("i16", shape, start, count, + adios2::ConstantDims, + m_TestData.I16.data()); + io.DefineVariable("i32", shape, start, count, + adios2::ConstantDims, + m_TestData.I32.data()); + io.DefineVariable("i64", shape, start, count, + adios2::ConstantDims, + m_TestData.I64.data()); + + io.DefineVariable("u8", shape, start, count, + adios2::ConstantDims, + m_TestData.U8.data()); + + io.DefineVariable("u16", shape, start, count, + adios2::ConstantDims, + m_TestData.U16.data()); + io.DefineVariable("u32", shape, start, count, + adios2::ConstantDims, + m_TestData.U32.data()); + io.DefineVariable("u64", shape, start, count, + adios2::ConstantDims, + m_TestData.U64.data()); + + io.DefineVariable("r32", shape, start, count, + adios2::ConstantDims, + m_TestData.R32.data()); + io.DefineVariable("r64", shape, start, count, + adios2::ConstantDims, + m_TestData.R64.data()); + } + + // Create the BP Engine + io.SetEngine("BPFile"); + io.SetParameter("Threads", "2"); + io.AddTransport("file"); + + adios2::Engine &bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + UpdateSmallTestData(m_TestData, static_cast(step), mpiRank, + mpiSize); + EXPECT_EQ(bpWriter.CurrentStep(), step); + bpWriter.WriteStep(); + } + + bpWriter.Close(); + } + + { + adios2::IO &io = adios.DeclareIO("ReadIO"); + io.SetParameter("Threads", "2"); + adios2::Engine &bpReader = io.Open(fname, adios2::Mode::Read); + + auto var_i8 = io.InquireVariable("i8"); + ASSERT_NE(var_i8, nullptr); + ASSERT_EQ(var_i8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i8->m_Shape[0], Ny); + ASSERT_EQ(var_i8->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i16 = io.InquireVariable("i16"); + ASSERT_NE(var_i16, nullptr); + ASSERT_EQ(var_i16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i16->m_Shape[0], Ny); + ASSERT_EQ(var_i16->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i32 = io.InquireVariable("i32"); + ASSERT_NE(var_i32, nullptr); + ASSERT_EQ(var_i32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i32->m_Shape[0], Ny); + ASSERT_EQ(var_i32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i64 = io.InquireVariable("i64"); + ASSERT_NE(var_i64, nullptr); + ASSERT_EQ(var_i64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i64->m_Shape[0], Ny); + ASSERT_EQ(var_i64->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u8 = io.InquireVariable("u8"); + ASSERT_NE(var_u8, nullptr); + ASSERT_EQ(var_u8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u8->m_Shape[0], Ny); + ASSERT_EQ(var_u8->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u16 = io.InquireVariable("u16"); + ASSERT_NE(var_u16, nullptr); + ASSERT_EQ(var_u16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u16->m_Shape[0], Ny); + ASSERT_EQ(var_u16->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u32 = io.InquireVariable("u32"); + ASSERT_NE(var_u32, nullptr); + ASSERT_EQ(var_u32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u32->m_Shape[0], Ny); + ASSERT_EQ(var_u32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u64 = io.InquireVariable("u64"); + ASSERT_NE(var_u64, nullptr); + ASSERT_EQ(var_u64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u64->m_Shape[0], Ny); + ASSERT_EQ(var_u64->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_r32 = io.InquireVariable("r32"); + ASSERT_NE(var_r32, nullptr); + ASSERT_EQ(var_r32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r32->m_Shape[0], Ny); + ASSERT_EQ(var_r32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_r64 = io.InquireVariable("r64"); + ASSERT_NE(var_r64, nullptr); + ASSERT_EQ(var_r64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r64->m_Shape[0], Ny); + ASSERT_EQ(var_r64->m_Shape[1], static_cast(mpiSize * Nx)); + + std::array I8; + std::array I16; + std::array I32; + std::array I64; + std::array U8; + std::array U16; + std::array U32; + std::array U64; + std::array R32; + std::array R64; + + const adios2::Dims start{0, static_cast(mpiRank * Nx)}; + const adios2::Dims count{Ny, Nx}; + + const adios2::Box sel(start, count); + + var_i8->SetSelection(sel); + var_i16->SetSelection(sel); + var_i32->SetSelection(sel); + var_i64->SetSelection(sel); + + var_u8->SetSelection(sel); + var_u16->SetSelection(sel); + var_u32->SetSelection(sel); + var_u64->SetSelection(sel); + + var_r32->SetSelection(sel); + var_r64->SetSelection(sel); + + unsigned int t = 0; + + while (bpReader.BeginStep() == adios2::StepStatus::OK) + { + const size_t currentStep = bpReader.CurrentStep(); + EXPECT_EQ(currentStep, static_cast(t)); + + bpReader.GetDeferred(*var_i8, I8.data()); + bpReader.GetDeferred(*var_i16, I16.data()); + bpReader.GetDeferred(*var_i32, I32.data()); + bpReader.GetDeferred(*var_i64, I64.data()); + + bpReader.GetDeferred(*var_u8, U8.data()); + bpReader.GetDeferred(*var_u16, U16.data()); + bpReader.GetDeferred(*var_u32, U32.data()); + bpReader.GetDeferred(*var_u64, U64.data()); + + bpReader.GetDeferred(*var_r32, R32.data()); + bpReader.GetDeferred(*var_r64, R64.data()); + + bpReader.EndStep(); + + // Generate test data for each rank uniquely + UpdateSmallTestData(m_OriginalData, static_cast(t), mpiRank, + mpiSize); + + for (size_t i = 0; i < Nx * Ny; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + EXPECT_EQ(I8[i], m_OriginalData.I8[i]) << msg; + EXPECT_EQ(I16[i], m_OriginalData.I16[i]) << msg; + EXPECT_EQ(I32[i], m_OriginalData.I32[i]) << msg; + EXPECT_EQ(I64[i], m_OriginalData.I64[i]) << msg; + EXPECT_EQ(U8[i], m_OriginalData.U8[i]) << msg; + EXPECT_EQ(U16[i], m_OriginalData.U16[i]) << msg; + EXPECT_EQ(U32[i], m_OriginalData.U32[i]) << msg; + EXPECT_EQ(U64[i], m_OriginalData.U64[i]) << msg; + EXPECT_EQ(R32[i], m_OriginalData.R32[i]) << msg; + EXPECT_EQ(R64[i], m_OriginalData.R64[i]) << msg; + } + ++t; + } + EXPECT_EQ(t, NSteps); + + bpReader.Close(); + } +} + +//****************************************************************************** +// 2D 4x2 test data +//****************************************************************************** + +TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, + ADIOS2BPWriteRead2D4x2MissingPerformGets) +{ + // Each process would write a 4x2 array and all processes would + // form a 2D 4 * (NumberOfProcess * Nx) matrix where Nx is 2 here + const std::string fname( + "ADIOS2BPWriteReadAsStream2D4x2MissingPerformGetsTest.bp"); + + int mpiRank = 0, mpiSize = 1; + // Number of rows + const std::size_t Nx = 2; + // Number of cols + const std::size_t Ny = 4; + + // Number of steps + const std::size_t NSteps = 3; + +#ifdef ADIOS2_HAVE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); + MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); +#endif + +// Write test data using ADIOS2 + +#ifdef ADIOS2_HAVE_MPI + adios2::ADIOS adios(MPI_COMM_WORLD, adios2::DebugON); +#else + adios2::ADIOS adios(true); +#endif + { + adios2::IO &io = adios.DeclareIO("TestIO"); + // Declare 2D variables (4 * (NumberOfProcess * Nx)) + // The local process' part (start, count) can be defined now or later + // before Write(). + { + adios2::Dims shape{static_cast(Ny), + static_cast(mpiSize * Nx)}; + adios2::Dims start{static_cast(0), + static_cast(mpiRank * Nx)}; + adios2::Dims count{static_cast(Ny), + static_cast(Nx)}; + + io.DefineVariable("i8", shape, start, count, + adios2::ConstantDims, + m_TestData.I8.data()); + io.DefineVariable("i16", shape, start, count, + adios2::ConstantDims, + m_TestData.I16.data()); + io.DefineVariable("i32", shape, start, count, + adios2::ConstantDims, + m_TestData.I32.data()); + io.DefineVariable("i64", shape, start, count, + adios2::ConstantDims, + m_TestData.I64.data()); + + io.DefineVariable("u8", shape, start, count, + adios2::ConstantDims, + m_TestData.U8.data()); + + io.DefineVariable("u16", shape, start, count, + adios2::ConstantDims, + m_TestData.U16.data()); + io.DefineVariable("u32", shape, start, count, + adios2::ConstantDims, + m_TestData.U32.data()); + io.DefineVariable("u64", shape, start, count, + adios2::ConstantDims, + m_TestData.U64.data()); + + io.DefineVariable("r32", shape, start, count, + adios2::ConstantDims, + m_TestData.R32.data()); + io.DefineVariable("r64", shape, start, count, + adios2::ConstantDims, + m_TestData.R64.data()); + } + + // Create the BP Engine + io.SetEngine("BPFile"); + io.SetParameter("Threads", "2"); + io.AddTransport("file"); + + adios2::Engine &bpWriter = io.Open(fname, adios2::Mode::Write); + + for (size_t step = 0; step < NSteps; ++step) + { + // Generate test data for each process uniquely + UpdateSmallTestData(m_TestData, static_cast(step), mpiRank, + mpiSize); + EXPECT_EQ(bpWriter.CurrentStep(), step); + bpWriter.WriteStep(); + } + + bpWriter.Close(); + } + + { + adios2::IO &io = adios.DeclareIO("ReadIO"); + io.SetParameter("Threads", "2"); + adios2::Engine &bpReader = io.Open(fname, adios2::Mode::Read); + + auto var_i8 = io.InquireVariable("i8"); + ASSERT_NE(var_i8, nullptr); + ASSERT_EQ(var_i8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i8->m_Shape[0], Ny); + ASSERT_EQ(var_i8->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i16 = io.InquireVariable("i16"); + ASSERT_NE(var_i16, nullptr); + ASSERT_EQ(var_i16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i16->m_Shape[0], Ny); + ASSERT_EQ(var_i16->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i32 = io.InquireVariable("i32"); + ASSERT_NE(var_i32, nullptr); + ASSERT_EQ(var_i32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i32->m_Shape[0], Ny); + ASSERT_EQ(var_i32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_i64 = io.InquireVariable("i64"); + ASSERT_NE(var_i64, nullptr); + ASSERT_EQ(var_i64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_i64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_i64->m_Shape[0], Ny); + ASSERT_EQ(var_i64->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u8 = io.InquireVariable("u8"); + ASSERT_NE(var_u8, nullptr); + ASSERT_EQ(var_u8->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u8->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u8->m_Shape[0], Ny); + ASSERT_EQ(var_u8->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u16 = io.InquireVariable("u16"); + ASSERT_NE(var_u16, nullptr); + ASSERT_EQ(var_u16->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u16->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u16->m_Shape[0], Ny); + ASSERT_EQ(var_u16->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u32 = io.InquireVariable("u32"); + ASSERT_NE(var_u32, nullptr); + ASSERT_EQ(var_u32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u32->m_Shape[0], Ny); + ASSERT_EQ(var_u32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_u64 = io.InquireVariable("u64"); + ASSERT_NE(var_u64, nullptr); + ASSERT_EQ(var_u64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_u64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_u64->m_Shape[0], Ny); + ASSERT_EQ(var_u64->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_r32 = io.InquireVariable("r32"); + ASSERT_NE(var_r32, nullptr); + ASSERT_EQ(var_r32->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r32->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r32->m_Shape[0], Ny); + ASSERT_EQ(var_r32->m_Shape[1], static_cast(mpiSize * Nx)); + + auto var_r64 = io.InquireVariable("r64"); + ASSERT_NE(var_r64, nullptr); + ASSERT_EQ(var_r64->m_ShapeID, adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64->m_AvailableStepsCount, NSteps); + ASSERT_EQ(var_r64->m_Shape[0], Ny); + ASSERT_EQ(var_r64->m_Shape[1], static_cast(mpiSize * Nx)); + + // If the size of the array is smaller than the data + // the result is weird... double and uint64_t would get + // completely garbage data + std::array I8; + std::array I16; + std::array I32; + std::array I64; + std::array U8; + std::array U16; + std::array U32; + std::array U64; + std::array R32; + std::array R64; + + const adios2::Dims start{0, static_cast(mpiRank * Nx)}; + const adios2::Dims count{Ny, Nx}; + + const adios2::Box sel(start, count); + + var_i8->SetSelection(sel); + var_i16->SetSelection(sel); + var_i32->SetSelection(sel); + var_i64->SetSelection(sel); + + var_u8->SetSelection(sel); + var_u16->SetSelection(sel); + var_u32->SetSelection(sel); + var_u64->SetSelection(sel); + + var_r32->SetSelection(sel); + var_r64->SetSelection(sel); + + unsigned int t = 0; + + while (bpReader.BeginStep() == adios2::StepStatus::OK) + { + const size_t currentStep = bpReader.CurrentStep(); + EXPECT_EQ(currentStep, static_cast(t)); + + bpReader.GetDeferred(*var_i8, I8.data()); + bpReader.GetDeferred(*var_i16, I16.data()); + bpReader.GetDeferred(*var_i32, I32.data()); + bpReader.GetDeferred(*var_i64, I64.data()); + + bpReader.GetDeferred(*var_u8, U8.data()); + bpReader.GetDeferred(*var_u16, U16.data()); + bpReader.GetDeferred(*var_u32, U32.data()); + bpReader.GetDeferred(*var_u64, U64.data()); + + bpReader.GetDeferred(*var_r32, R32.data()); + bpReader.GetDeferred(*var_r64, R64.data()); + + bpReader.EndStep(); + + // Generate test data for each rank uniquely + UpdateSmallTestData(m_OriginalData, static_cast(t), mpiRank, + mpiSize); + + for (size_t i = 0; i < Nx * Ny; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i << " rank=" << mpiRank; + std::string msg = ss.str(); + + EXPECT_EQ(I8[i], m_OriginalData.I8[i]) << msg; + EXPECT_EQ(I16[i], m_OriginalData.I16[i]) << msg; + EXPECT_EQ(I32[i], m_OriginalData.I32[i]) << msg; + EXPECT_EQ(I64[i], m_OriginalData.I64[i]) << msg; + EXPECT_EQ(U8[i], m_OriginalData.U8[i]) << msg; + EXPECT_EQ(U16[i], m_OriginalData.U16[i]) << msg; + EXPECT_EQ(U32[i], m_OriginalData.U32[i]) << msg; + EXPECT_EQ(U64[i], m_OriginalData.U64[i]) << msg; + EXPECT_EQ(R32[i], m_OriginalData.R32[i]) << msg; + EXPECT_EQ(R64[i], m_OriginalData.R64[i]) << msg; + } + ++t; + } + EXPECT_EQ(t, NSteps); + bpReader.Close(); + } +} + +//****************************************************************************** +// main +//****************************************************************************** + +int main(int argc, char **argv) +{ +#ifdef ADIOS2_HAVE_MPI + MPI_Init(nullptr, nullptr); +#endif + + ::testing::InitGoogleTest(&argc, argv); + int result = RUN_ALL_TESTS(); + +#ifdef ADIOS2_HAVE_MPI + MPI_Finalize(); +#endif + + return result; +}