Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bug BP5 Get for chunks in an ND array #3133

Merged
merged 2 commits into from
Mar 25, 2022
Merged

Bug BP5 Get for chunks in an ND array #3133

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
42ae299
Testing for different reading patterns with GPU buffers
anagainaru Mar 25, 2022
e6a8f01
GPU copy for BP5 capable of copying mutiple chunks
anagainaru Mar 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 11 additions & 18 deletions source/adios2/helper/adiosMemory.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -451,8 +451,9 @@ int NdCopy(const char *in, const Dims &inStart, const Dims &inCount,
#ifdef ADIOS2_HAVE_CUDA
if (MemSpace == MemorySpace::CUDA)
{
helper::CudaMemCopyFromBuffer(outOvlpBase, 0, inOvlpBase,
blockSize);
helper::NdCopyCUDA(inOvlpBase, outOvlpBase, inOvlpGapSize,
outOvlpGapSize, ovlpCount, minContDim,
blockSize);
return 0;
}
#endif
Expand Down Expand Up @@ -509,6 +510,14 @@ int NdCopy(const char *in, const Dims &inStart, const Dims &inCount,
// padding
else
{
#ifdef ADIOS2_HAVE_CUDA
if (MemSpace == MemorySpace::CUDA)
{
helper::Throw<std::invalid_argument>(
"Helper", "Memory", "CopyContiguousMemory",
"Direct byte order reversal not supported for GPU buffers");
}
#endif
// Dims revInCount(inCount);
// Dims revOutCount(outCount);
//
Expand Down Expand Up @@ -607,14 +616,6 @@ int NdCopy(const char *in, const Dims &inStart, const Dims &inCount,
// Same Endian"
if (inIsLittleEndian == outIsLittleEndian)
{
#ifdef ADIOS2_HAVE_CUDA
if (MemSpace == MemorySpace::CUDA)
{
helper::CudaMemCopyFromBuffer(outOvlpBase, 0, inOvlpBase,
blockSize);
return 0;
}
#endif
if (!safeMode)
{
NdCopyRecurDFNonSeqDynamic(0, inOvlpBase, outOvlpBase,
Expand All @@ -632,14 +633,6 @@ int NdCopy(const char *in, const Dims &inStart, const Dims &inCount,
// different Endian"
else
{
#ifdef ADIOS2_HAVE_CUDA
if (MemSpace == MemorySpace::CUDA)
{
helper::Throw<std::invalid_argument>(
"Helper", "Memory", "CopyContiguousMemory",
"Direct byte order reversal not supported for GPU buffers");
}
#endif
if (!safeMode)
{
NdCopyRecurDFNonSeqDynamicRevEndian(
Expand Down
31 changes: 31 additions & 0 deletions source/adios2/helper/adiosMemory.inl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,37 @@ void CudaMemCopyFromBuffer(T *GPUbuffer, size_t position, const char *source,
char *dest = reinterpret_cast<char *>(GPUbuffer);
MemcpyBufferToGPU(dest, source + position, size);
}

static inline void NdCopyCUDA(const char *&inOvlpBase, char *&outOvlpBase,
Dims &inOvlpGapSize, Dims &outOvlpGapSize,
Dims &ovlpCount, size_t minContDim,
size_t blockSize)
{
Dims pos(ovlpCount.size(), 0);
size_t curDim = 0;
while (true)
{
while (curDim != minContDim)
{
pos[curDim]++;
curDim++;
}
CudaMemCopyFromBuffer(outOvlpBase, 0, inOvlpBase, blockSize);
inOvlpBase += blockSize;
outOvlpBase += blockSize;
do
{
if (curDim == 0)
{
return;
}
inOvlpBase += inOvlpGapSize[curDim];
outOvlpBase += outOvlpGapSize[curDim];
pos[curDim] = 0;
curDim--;
} while (pos[curDim] == ovlpCount[curDim]);
}
}
#endif

template <class T>
Expand Down
24 changes: 20 additions & 4 deletions testing/adios2/engine/bp/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,14 +164,30 @@ gtest_add_tests_helper(StepsInSituLocalArray MPI_ALLOW BP Engine.BP. .FileStream
WORKING_DIRECTORY ${FS_DIR} EXTRA_ARGS "FileStream"
)

if(ADIOS2_HAVE_CUDA AND ADIOS2_HAVE_BP5)
if(ADIOS2_HAVE_CUDA)
enable_language(CUDA)
gtest_add_tests_helper(WriteReadCuda MPI_ALLOW BP Engine.BP. .BP5
WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5"
)
gtest_add_tests_helper(WriteReadCuda MPI_ALLOW BP Engine.BP. .BP4
WORKING_DIRECTORY ${BP4_DIR} EXTRA_ARGS "BP4"
)
gtest_add_tests_helper(SelectionsCuda MPI_ALLOW BP Engine.BP. .BP4
WORKING_DIRECTORY ${BP4_DIR} EXTRA_ARGS "BP4"
)
if(ADIOS2_HAVE_BP5)
gtest_add_tests_helper(WriteReadCuda MPI_ALLOW BP Engine.BP. .BP5
WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5"
)
gtest_add_tests_helper(SelectionsCuda MPI_ALLOW BP Engine.BP. .BP5
WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5"
)
endif()

foreach(tgt ${Test.Engine.BP.WriteReadCuda-TARGETS})
target_sources(${tgt} PRIVATE operations/CudaRoutines.cu)
target_link_libraries(${tgt} CUDA::cudart)
endforeach()

foreach(tgt ${Test.Engine.BP.SelectionsCuda-TARGETS})
target_sources(${tgt} PRIVATE operations/CudaRoutines.cu)
target_link_libraries(${tgt} CUDA::cudart)
endforeach()
endif()
Loading