Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

More BP5 Tests #2989

Merged
merged 3 commits into from
Jan 4, 2022
Merged

More BP5 Tests #2989

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6e49abe
More BP5 Tests
eisenhauer Jan 3, 2022
cd25747
Get metadata from other writer ranks
eisenhauer Jan 4, 2022
e9693a7
warning
eisenhauer Jan 4, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions source/adios2/engine/bp5/BP5Writer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1212,6 +1212,11 @@ void BP5Writer::PopulateMetadataIndexFileContent(
position += 8;
}

size_t BP5Writer::DebugGetDataBufferSize() const
{
return m_BP5Serializer.DebugGetDataBufferSize();
}

} // end namespace engine
} // end namespace core
} // end namespace adios2
2 changes: 2 additions & 0 deletions source/adios2/engine/bp5/BP5Writer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ class BP5Writer : public BP5Engine, public core::Engine
void EndStep() final;
void Flush(const int transportIndex = -1) final;

size_t DebugGetDataBufferSize() const final;

private:
/** Single object controlling BP buffering */
format::BP5Serializer m_BP5Serializer;
Expand Down
19 changes: 12 additions & 7 deletions source/adios2/toolkit/format/bp5/BP5Deserializer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -859,15 +859,15 @@ bool BP5Deserializer::QueueGet(core::VariableBase &variable, void *DestData)
}
}

void BP5Deserializer::GetSingleValueFromMetadata(core::VariableBase &variable,
bool BP5Deserializer::GetSingleValueFromMetadata(core::VariableBase &variable,
BP5VarRec *VarRec,
void *DestData, size_t Step,
size_t WriterRank)
{
char *src = (char *)GetMetadataBase(VarRec, Step, WriterRank);

if (!src)
return;
return false;

if (variable.m_SelectionType == adios2::SelectionType::WriteBlock)
WriterRank = variable.m_BlockID;
Expand All @@ -881,6 +881,7 @@ void BP5Deserializer::GetSingleValueFromMetadata(core::VariableBase &variable,
std::string *TmpStr = static_cast<std::string *>(DestData);
TmpStr->assign(*(const char **)src);
}
return true;
}

bool BP5Deserializer::QueueGetSingle(core::VariableBase &variable,
Expand All @@ -889,9 +890,13 @@ bool BP5Deserializer::QueueGetSingle(core::VariableBase &variable,
BP5VarRec *VarRec = VarByKey[&variable];
if (VarRec->OrigShapeID == ShapeID::GlobalValue)
{
int WriterRank = 0;
GetSingleValueFromMetadata(variable, VarRec, DestData, Step,
WriterRank);
for (size_t WriterRank = 0; WriterRank < m_WriterCohortSize;
WriterRank++)
{
if (GetSingleValueFromMetadata(variable, VarRec, DestData, Step,
WriterRank))
return false;
}
return false;
}
if (VarRec->OrigShapeID == ShapeID::LocalValue)
Expand All @@ -902,8 +907,8 @@ bool BP5Deserializer::QueueGetSingle(core::VariableBase &variable,
WriterRank < variable.m_Count[0] + variable.m_Start[0];
WriterRank++)
{
GetSingleValueFromMetadata(variable, VarRec, DestData, Step,
WriterRank);
(void)GetSingleValueFromMetadata(variable, VarRec, DestData, Step,
WriterRank);
DestData = (char *)DestData +
variable.m_ElementSize; // use variable.m_ElementSize
// because it's the size in local
Expand Down
2 changes: 1 addition & 1 deletion source/adios2/toolkit/format/bp5/BP5Deserializer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ class BP5Deserializer : virtual public BP5Base
const size_t *LocalOffsets, size_t *LocalIndex);
size_t RelativeToAbsoluteStep(const BP5VarRec *VarRec, size_t RelStep);
int FindOffset(size_t Dims, const size_t *Size, const size_t *Index);
void GetSingleValueFromMetadata(core::VariableBase &variable,
bool GetSingleValueFromMetadata(core::VariableBase &variable,
BP5VarRec *VarRec, void *DestData,
size_t Step, size_t WriterRank);
void ExtractSelectionFromPartialRM(int ElementSize, size_t Dims,
Expand Down
7 changes: 7 additions & 0 deletions source/adios2/toolkit/format/bp5/BP5Serializer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1144,5 +1144,12 @@ void *BP5Serializer::GetPtr(int bufferIdx, size_t posInBuffer)
return CurDataBuffer->GetPtr(bufferIdx, posInBuffer);
}

size_t BP5Serializer::DebugGetDataBufferSize() const
{
if (CurDataBuffer == NULL)
return 0;
return CurDataBuffer->Size();
}

} // end namespace format
} // end namespace adios2
2 changes: 2 additions & 0 deletions source/adios2/toolkit/format/bp5/BP5Serializer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,8 @@ class BP5Serializer : virtual public BP5Base
void *GetPtr(int bufferIdx, size_t posInBuffer);
size_t CalcSize(const size_t Count, const size_t *Vals);

size_t DebugGetDataBufferSize() const;

int m_StatsLevel = 1;

private:
Expand Down
6 changes: 3 additions & 3 deletions testing/adios2/engine/bp/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ bp3_bp4_gtest_add_tests_helper(WriteFlushRead MPI_ALLOW)
bp3_bp4_gtest_add_tests_helper(WriteMultiblockRead MPI_ALLOW)
bp3_bp4_gtest_add_tests_helper(WriteReadMultiblock MPI_ALLOW)
bp_gtest_add_tests_helper(WriteReadVector MPI_ALLOW)
bp3_bp4_gtest_add_tests_helper(WriteReadAttributesMultirank MPI_ALLOW)
bp_gtest_add_tests_helper(WriteReadAttributesMultirank MPI_ALLOW)
bp_gtest_add_tests_helper(LargeMetadata MPI_ALLOW)
if(ADIOS2_HAVE_BP5)
set (BP5LargeMeta "Engine.BP.BPLargeMetadata.BPWrite1D_LargeMetadata.BP5.Serial")
Expand All @@ -96,13 +96,13 @@ bp3_bp4_gtest_add_tests_helper(TimeAggregation MPI_ALLOW)
bp_gtest_add_tests_helper(NoXMLRecovery MPI_ALLOW)
bp3_bp4_gtest_add_tests_helper(StepsFileGlobalArray MPI_ALLOW)
bp3_bp4_gtest_add_tests_helper(StepsFileLocalArray MPI_ALLOW)
bp3_bp4_gtest_add_tests_helper(SelectSteps MPI_ALLOW)
bp_gtest_add_tests_helper(SelectSteps MPI_ALLOW)

bp3_bp4_gtest_add_tests_helper(SelectionsOnRowMajorData MPI_NONE)
bp3_bp4_gtest_add_tests_helper(SelectionsOnColumnMajorData MPI_NONE)

if(NOT MSVC)
bp3_bp4_gtest_add_tests_helper(BufferSize MPI_NONE)
bp_gtest_add_tests_helper(BufferSize MPI_NONE)
endif()

if(ADIOS2_HAVE_MPI)
Expand Down
52 changes: 45 additions & 7 deletions testing/adios2/engine/bp/TestBPStepsFileGlobalArray.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,13 +159,13 @@ TEST_P(BPStepsFileGlobalArrayReaders, EveryStep)
{
io.SetEngine(engineName);
}
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

if (readMode == ReadMode::ReadFileAll)
{
/// Read back data with File reading mode
/// Read back the whole thing and check data
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

if (!mpiRank)
{
std::cout << "Read with File reading mode, read all steps at once"
Expand Down Expand Up @@ -210,6 +210,9 @@ TEST_P(BPStepsFileGlobalArrayReaders, EveryStep)
{
/// Read back data with File reading mode
/// Read back step by step and check data
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

if (!mpiRank)
{
std::cout << "Read with File reading mode, read step by step"
Expand Down Expand Up @@ -239,6 +242,9 @@ TEST_P(BPStepsFileGlobalArrayReaders, EveryStep)
{
/// Read back data with File reading mode
/// Read back step by step and block by block and check data
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

if (!mpiRank)
{
std::cout << "Read with File reading mode, read step by step, "
Expand Down Expand Up @@ -274,6 +280,9 @@ TEST_P(BPStepsFileGlobalArrayReaders, EveryStep)
{
/// Read back data with Stream reading mode
/// Read back step by step and check data
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

if (!mpiRank)
{
std::cout << "Read with Stream reading mode, read step by step"
Expand Down Expand Up @@ -302,6 +311,9 @@ TEST_P(BPStepsFileGlobalArrayReaders, EveryStep)
}
else if (readMode == ReadMode::ReadStreamBlocks)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

/// Read back data with Stream reading mode
/// Read back step by step and check data
if (!mpiRank)
Expand Down Expand Up @@ -412,11 +424,12 @@ TEST_P(BPStepsFileGlobalArrayReaders, NewVarPerStep)
{
io.SetEngine(engineName);
}
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

if (readMode == ReadMode::ReadFileAll)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

/// Read back each variable with File reading mode
/// Use SetStepSelection(0,1) explicitly
if (!mpiRank)
Expand Down Expand Up @@ -459,6 +472,9 @@ TEST_P(BPStepsFileGlobalArrayReaders, NewVarPerStep)
}
else if (readMode == ReadMode::ReadFileStepByStep)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

/// Read back each variable with File reading mode
/// and do not use SetStepSelection() so default read after open is
/// tested
Expand Down Expand Up @@ -490,6 +506,9 @@ TEST_P(BPStepsFileGlobalArrayReaders, NewVarPerStep)
}
else if (readMode == ReadMode::ReadFileStepByStepBlocks)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

/// Read back each variable with File reading mode
/// Read back block by block and check data
if (!mpiRank)
Expand Down Expand Up @@ -527,6 +546,9 @@ TEST_P(BPStepsFileGlobalArrayReaders, NewVarPerStep)
}
else if (readMode == ReadMode::ReadStream)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

/// Read back each variable with Streaming reading mode
if (!mpiRank)
{
Expand Down Expand Up @@ -561,6 +583,9 @@ TEST_P(BPStepsFileGlobalArrayReaders, NewVarPerStep)
}
else if (readMode == ReadMode::ReadStreamBlocks)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

/// Read back each variable with Streaming reading mode
if (!mpiRank)
{
Expand Down Expand Up @@ -699,11 +724,12 @@ TEST_P(BPStepsFileGlobalArrayParameters, EveryOtherStep)
{
io.SetEngine(engineName);
}
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

if (readMode == ReadMode::ReadFileAll)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

/// Read back data with File reading mode
/// Read back the whole thing and check data
if (!mpiRank)
Expand Down Expand Up @@ -736,6 +762,9 @@ TEST_P(BPStepsFileGlobalArrayParameters, EveryOtherStep)
}
else if (readMode == ReadMode::ReadFileStepByStep)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

/// Read back data with File reading mode
/// Read back step by step and check data
if (!mpiRank)
Expand Down Expand Up @@ -767,6 +796,9 @@ TEST_P(BPStepsFileGlobalArrayParameters, EveryOtherStep)
}
else if (readMode == ReadMode::ReadFileStepByStepBlocks)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

/// Read back data with File reading mode
/// Read back step by step, block by block and check data
if (!mpiRank)
Expand Down Expand Up @@ -804,6 +836,9 @@ TEST_P(BPStepsFileGlobalArrayParameters, EveryOtherStep)
}
else if (readMode == ReadMode::ReadStream)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

/// Read back data with Stream reading mode
/// Read back step by step and check data
if (!mpiRank)
Expand Down Expand Up @@ -845,6 +880,9 @@ TEST_P(BPStepsFileGlobalArrayParameters, EveryOtherStep)
}
else if (readMode == ReadMode::ReadStreamBlocks)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

/// Read back data with Stream reading mode
/// Read back step by step and check data
if (!mpiRank)
Expand Down
27 changes: 18 additions & 9 deletions testing/adios2/engine/bp/TestBPStepsFileLocalArray.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,13 +151,13 @@ TEST_P(BPStepsFileLocalArrayReaders, EveryStep)
{
io.SetEngine(engineName);
}
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

if (readMode == ReadMode::ReadFileStepByStepBlocks)
{
/// Read back data with File reading mode
/// Read back step by step and block by block and check data
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

if (!mpiRank)
{
std::cout << "Read with File reading mode, read step by step, "
Expand Down Expand Up @@ -194,6 +194,9 @@ TEST_P(BPStepsFileLocalArrayReaders, EveryStep)
{
/// Read back data with Stream reading mode
/// Read back step by step and check data
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

if (!mpiRank)
{
std::cout << "Read with Stream reading mode, read step by step, "
Expand Down Expand Up @@ -303,13 +306,13 @@ TEST_P(BPStepsFileLocalArrayReaders, NewVarPerStep)
{
io.SetEngine(engineName);
}
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

if (readMode == ReadMode::ReadFileStepByStepBlocks)
{
/// Read back each variable with File reading mode
/// Read back block by block and check data
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

if (!mpiRank)
{
std::cout
Expand Down Expand Up @@ -347,6 +350,9 @@ TEST_P(BPStepsFileLocalArrayReaders, NewVarPerStep)
else if (readMode == ReadMode::ReadStreamBlocks)
{
/// Read back each variable with Streaming reading mode
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

if (!mpiRank)
{
std::cout
Expand Down Expand Up @@ -482,11 +488,11 @@ TEST_P(BPStepsFileLocalArrayParameters, EveryOtherStep)
{
io.SetEngine(engineName);
}
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

if (readMode == ReadMode::ReadFileStepByStepBlocks)
{
adios2::Engine engine = io.Open(fname, adios2::Mode::ReadRandomAccess);
EXPECT_TRUE(engine);

/// Read back data with File reading mode
/// Read back step by step, block by block and check data
if (!mpiRank)
Expand Down Expand Up @@ -527,6 +533,9 @@ TEST_P(BPStepsFileLocalArrayParameters, EveryOtherStep)
{
/// Read back data with Stream reading mode
/// Read back step by step and check data
adios2::Engine engine = io.Open(fname, adios2::Mode::Read);
EXPECT_TRUE(engine);

if (!mpiRank)
{
std::cout
Expand Down
3 changes: 2 additions & 1 deletion testing/adios2/engine/bp/TestBPWriteReadAttributesMultirank.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,8 @@ TEST_F(BPWriteReadAttributeTestMultirank, ADIOS2BPWriteReadArrayTypes)
ioRead.SetEngine(engineName);
}

adios2::Engine bpRead = ioRead.Open(fName, adios2::Mode::Read);
adios2::Engine bpRead =
ioRead.Open(fName, adios2::Mode::ReadRandomAccess);

auto var = ioRead.InquireVariable<int>(varpath);
EXPECT_TRUE(var);
Expand Down