Skip to content

Commit

Permalink
Merge pull request ornladios#3823 from eisenhauer/SstMemSel
Browse files Browse the repository at this point in the history 
Add Memory Selection to SST, and prototype a mechanism for running st…

(cherry picked from commit c503940)
  • Loading branch information
@eisenhauer @vicentebolea
eisenhauer authored and vicentebolea committed Oct 24, 2023
1 parent 15326e6 commit 377a7d5
Show file tree
Hide file tree
Showing 4 changed files with 159 additions and 36 deletions.
62 changes: 51 additions & 11 deletions source/adios2/engine/sst/SstWriter.tcc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,21 +71,61 @@ void SstWriter::PutSyncCommon(Variable<T> &variable, const T *values)
}
else
{
if (variable.m_Type == DataType::String)
if (!variable.m_MemoryCount.empty())
{
std::string &source = *(std::string *)values;
void *p = &(source[0]);
m_BP5Serializer->Marshal(
(void *)&variable, variable.m_Name.c_str(), variable.m_Type,
variable.m_ElementSize, DimCount, Shape, Count, Start, &p,
true, nullptr);
size_t ObjSize;
if (variable.m_Type == DataType::Struct)
{
ObjSize = variable.m_ElementSize;
}
else
{
ObjSize = helper::GetDataTypeSize(variable.m_Type);
}

const bool sourceRowMajor = helper::IsRowMajor(m_IO.m_HostLanguage);
helper::DimsArray MemoryStart(variable.m_MemoryStart);
helper::DimsArray MemoryCount(variable.m_MemoryCount);
helper::DimsArray varCount(variable.m_Count);

int DimCount = (int)variable.m_Count.size();
helper::DimsArray ZeroDims(DimCount, (size_t)0);
// get a temporary span then fill with memselection now
format::BufferV::BufferPos bp5span(0, 0, 0);

m_BP5Serializer->Marshal((void *)&variable, variable.m_Name.c_str(),
variable.m_Type, variable.m_ElementSize, DimCount, Shape,
Count, Start, nullptr, false, &bp5span);
void *ptr = m_BP5Serializer->GetPtr(bp5span.bufferIdx, bp5span.posInBuffer);

if (!sourceRowMajor)
{
std::reverse(MemoryStart.begin(), MemoryStart.end());
std::reverse(MemoryCount.begin(), MemoryCount.end());
std::reverse(varCount.begin(), varCount.end());
}
helper::NdCopy((const char *)values, helper::CoreDims(ZeroDims), MemoryCount,
sourceRowMajor, false, (char *)ptr, MemoryStart, varCount,
sourceRowMajor, false, (int)ObjSize, helper::CoreDims(),
helper::CoreDims(), helper::CoreDims(), helper::CoreDims(),
false /* safemode */, variable.m_MemSpace);
}
else
{
m_BP5Serializer->Marshal(
(void *)&variable, variable.m_Name.c_str(), variable.m_Type,
variable.m_ElementSize, DimCount, Shape, Count, Start,
values, true, nullptr);
if (variable.m_Type == DataType::String)
{
std::string &source = *(std::string *)values;
void *p = &(source[0]);
m_BP5Serializer->Marshal((void *)&variable, variable.m_Name.c_str(),
variable.m_Type, variable.m_ElementSize, DimCount,
Shape, Count, Start, &p, true, nullptr);
}
else
{
m_BP5Serializer->Marshal((void *)&variable, variable.m_Name.c_str(),
variable.m_Type, variable.m_ElementSize, DimCount,
Shape, Count, Start, values, true, nullptr);
}
}
}
}
Expand Down
116 changes: 91 additions & 25 deletions testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
#include "../SmallTestData.h"

std::string engineName; // comes from command line
bool DoWrite = true;
bool DoRead = true;

namespace
{
Expand Down Expand Up @@ -178,9 +180,13 @@ void AssignStep3D(const size_t step, std::vector<std::complex<double>> &vector,

} // end anonymous namespace

#if ADIOS2_USE_MPI
MPI_Comm testComm;
#endif

void BPSteps1D(const size_t ghostCells)
{
const std::string fname("BPSteps1D_" + std::to_string(ghostCells) + ".bp");
const std::string fname("BPSteps1D_" + std::to_string(ghostCells));

int mpiRank = 0, mpiSize = 1;
// Number of rows
Expand All @@ -190,15 +196,16 @@ void BPSteps1D(const size_t ghostCells)
const size_t NSteps = 3;

#if ADIOS2_USE_MPI
MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
MPI_Comm_rank(testComm, &mpiRank);
MPI_Comm_size(testComm, &mpiSize);
#endif

#if ADIOS2_USE_MPI
adios2::ADIOS adios(MPI_COMM_WORLD);
adios2::ADIOS adios(testComm);
#else
adios2::ADIOS adios;
#endif
if (DoWrite)
{
adios2::IO io = adios.DeclareIO("WriteIO");

Expand All @@ -207,6 +214,7 @@ void BPSteps1D(const size_t ghostCells)
io.SetEngine(engineName);
}

io.SetParameters("StatsLevel=1");
const adios2::Dims shape{static_cast<size_t>(Nx * mpiSize)};
const adios2::Dims start{static_cast<size_t>(Nx * mpiRank)};
const adios2::Dims count{Nx};
Expand Down Expand Up @@ -272,9 +280,10 @@ void BPSteps1D(const size_t ghostCells)
bpWriter.Close();
}
#if ADIOS2_USE_MPI
MPI_Barrier(MPI_COMM_WORLD);
MPI_Barrier(testComm);
#endif
// Reader
if (DoRead)
{
adios2::IO io = adios.DeclareIO("ReadIO");

Expand Down Expand Up @@ -403,8 +412,7 @@ void BPSteps1D(const size_t ghostCells)

void BPSteps2D4x2(const size_t ghostCells)
{
const std::string fname("BPSteps2D4x2_" + std::to_string(ghostCells) +
".bp");
const std::string fname("BPSteps2D4x2_" + std::to_string(ghostCells));

int mpiRank = 0, mpiSize = 1;
// Number of rows
Expand All @@ -418,15 +426,16 @@ void BPSteps2D4x2(const size_t ghostCells)
const size_t NSteps = 3;

#if ADIOS2_USE_MPI
MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
MPI_Comm_rank(testComm, &mpiRank);
MPI_Comm_size(testComm, &mpiSize);
#endif

#if ADIOS2_USE_MPI
adios2::ADIOS adios(MPI_COMM_WORLD);
adios2::ADIOS adios(testComm);
#else
adios2::ADIOS adios;
#endif
if (DoWrite)
{
adios2::IO io = adios.DeclareIO("WriteIO");

Expand Down Expand Up @@ -500,9 +509,10 @@ void BPSteps2D4x2(const size_t ghostCells)
bpWriter.Close();
}
#if ADIOS2_USE_MPI
MPI_Barrier(MPI_COMM_WORLD);
MPI_Barrier(testComm);
#endif
// Reader
if (DoRead)
{
adios2::IO io = adios.DeclareIO("ReadIO");

Expand Down Expand Up @@ -641,8 +651,7 @@ void BPSteps2D4x2(const size_t ghostCells)

void BPSteps3D8x2x4(const size_t ghostCells)
{
const std::string fname("BPSteps3D8x2x4_" + std::to_string(ghostCells) +
".bp");
const std::string fname("BPSteps3D8x2x4_" + std::to_string(ghostCells));

int mpiRank = 0, mpiSize = 1;
// Number of rows
Expand All @@ -658,15 +667,16 @@ void BPSteps3D8x2x4(const size_t ghostCells)
const size_t NSteps = 3;

#if ADIOS2_USE_MPI
MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
MPI_Comm_rank(testComm, &mpiRank);
MPI_Comm_size(testComm, &mpiSize);
#endif

#if ADIOS2_USE_MPI
adios2::ADIOS adios(MPI_COMM_WORLD);
adios2::ADIOS adios(testComm);
#else
adios2::ADIOS adios;
#endif
if (DoWrite)
{
adios2::IO io = adios.DeclareIO("WriteIO");

Expand Down Expand Up @@ -750,9 +760,10 @@ void BPSteps3D8x2x4(const size_t ghostCells)
bpWriter.Close();
}
#if ADIOS2_USE_MPI
MPI_Barrier(MPI_COMM_WORLD);
MPI_Barrier(testComm);
#endif
// Reader
if (DoRead)
{
adios2::IO io = adios.DeclareIO("ReadIO");

Expand Down Expand Up @@ -950,25 +961,80 @@ INSTANTIATE_TEST_SUITE_P(ghostCells, BPWriteMemSelReadVector,

int main(int argc, char **argv)
{
int result;
::testing::InitGoogleTest(&argc, argv);
int bare_arg = 0;

for (int i = 1; i < argc; i++)
{
if (strcmp(argv[i], "-do_write") == 0)
{
DoWrite = true;
DoRead = false;
}
else if (strcmp(argv[i], "-do_read") == 0)
{
DoWrite = false;
DoRead = true;
}
else if (argv[i][0] == '-')
{
std::cerr << "Unknown argument: " << argv[i] << std::endl;
exit(1);
}
else
{
std::string fname;
std::string engineParams;
if (bare_arg == 0)
{
/* first arg without -- is engine */
engineName = std::string(argv[1]);
bare_arg++;
}
else if (bare_arg == 1)
{
/* second arg without -- is filename */
// fname = std::string(argv[1]);
bare_arg++;
}
else if (bare_arg == 2)
{
// engineParams = ParseEngineParams(argv[1]);
bare_arg++;
}
else
{

throw std::invalid_argument("Unknown argument \"" + std::string(argv[1]) + "\"");
}
}
}

#if ADIOS2_USE_MPI
int provided;
int thread_support_level =
(engineName == "SST" || engineName == "sst") ? MPI_THREAD_MULTIPLE : MPI_THREAD_SINGLE;

// MPI_THREAD_MULTIPLE is only required if you enable the SST MPI_DP
MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided);
#endif
MPI_Init_thread(nullptr, nullptr, thread_support_level, &provided);

int result;
::testing::InitGoogleTest(&argc, argv);
int key;
MPI_Comm_rank(MPI_COMM_WORLD, &key);

if (argc > 1)
{
engineName = std::string(argv[1]);
}
const unsigned int color = (DoRead & !DoWrite) ? 1 : 0;

MPI_Comm_split(MPI_COMM_WORLD, color, key, &testComm);
#endif

result = RUN_ALL_TESTS();

#if ADIOS2_USE_MPI
#ifdef CRAY_MPICH_VERSION
MPI_Barrier(MPI_COMM_WORLD);
#else
MPI_Finalize();
#endif
#endif

return result;
Expand Down
5 changes: 5 additions & 0 deletions testing/adios2/engine/staging-common/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,8 +149,13 @@ set (ALL_SIMPLE_TESTS "")
list (APPEND ALL_SIMPLE_TESTS ${SIMPLE_TESTS} ${SIMPLE_FORTRAN_TESTS} ${SIMPLE_MPI_TESTS} ${SIMPLE_ZFP_TESTS})

set (SST_SPECIFIC_TESTS "")
import_bp_test(WriteMemorySelectionRead 1 1)
list (APPEND SST_SPECIFIC_TESTS "WriteMemorySelectionRead.1x1")
list (APPEND SST_SPECIFIC_TESTS "1x1.SstRUDP;1x1.LocalMultiblock;RoundRobinDistribution.1x1x3;AllToAllDistribution.1x1x3;OnDemandSingle.1x1")

if (ADIOS2_HAVE_MPI)
import_bp_test(WriteMemorySelectionRead 3 3)
list (APPEND SST_SPECIFIC_TESTS "WriteMemorySelectionRead.3x3")
list (APPEND SST_SPECIFIC_TESTS "2x3.SstRUDP;2x1.LocalMultiblock;5x3.LocalMultiblock;")
endif()

Expand Down
12 changes: 12 additions & 0 deletions testing/adios2/engine/staging-common/TestSupp.cmake
View file
Original file line number Diff line number Diff line change
Expand Up @@ -321,3 +321,15 @@ function(from_hex HEX DEC)
set(${DEC} ${_res} PARENT_SCOPE)
endfunction()

function(import_bp_test BASENAME WRITE_SCALE READ_SCALE)
set (WRITER_POSTFIX "Serial")
set (READER_POSTFIX "Serial")
if(ADIOS2_HAVE_MPI)
set (WRITER_POSTFIX "MPI")
endif()
if(ADIOS2_HAVE_MPI)
set (READER_POSTFIX "MPI")
endif()
set (${BASENAME}.${WRITE_SCALE}x${READ_SCALE}_CMD "run_test.py.$<CONFIG> -nw ${WRITE_SCALE} -nr ${READ_SCALE} --warg=-do_write --rarg=-do_read -w $<TARGET_FILE:Test.Engine.BP.${BASENAME}.${WRITER_POSTFIX}> -r $<TARGET_FILE:Test.Engine.BP.${BASENAME}.${READER_POSTFIX}>" PARENT_SCOPE)

endfunction()

0 comments on commit 377a7d5

Please sign in to comment.