diff --git a/CMakeLists.txt b/CMakeLists.txt index b7f0f44a1d..1268116008 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -222,7 +222,7 @@ endif() set(ADIOS2_CONFIG_OPTS - BP5 DataMan DataSpaces HDF5 HDF5_VOL MHS SST CUDA Fortran MPI Python Blosc BZip2 LIBPRESSIO MGARD PNG SZ ZFP DAOS IME O_DIRECT SysVShMem ZeroMQ Profiling Endian_Reverse Sodium + BP5 DataMan DataSpaces HDF5 HDF5_VOL MHS SST CUDA Fortran MPI Python Blosc BZip2 LIBPRESSIO MGARD PNG SZ ZFP DAOS IME O_DIRECT Sodium SysVShMem ZeroMQ Profiling Endian_Reverse ) GenerateADIOSHeaderConfig(${ADIOS2_CONFIG_OPTS}) diff --git a/bindings/CXX11/adios2/cxx11/Attribute.cpp b/bindings/CXX11/adios2/cxx11/Attribute.cpp index 23a31754e1..0627a8c21a 100644 --- a/bindings/CXX11/adios2/cxx11/Attribute.cpp +++ b/bindings/CXX11/adios2/cxx11/Attribute.cpp @@ -64,6 +64,14 @@ namespace adios2 } \ \ template <> \ + bool Attribute::IsValue() const \ + { \ + helper::CheckForNullptr(m_Attribute, \ + "in call to Attribute::IsValue()"); \ + return m_Attribute->m_IsSingleValue; \ + } \ + \ + template <> \ std::string ToString(const Attribute &attribute) \ { \ return std::string("Attribute<") + attribute.Type() + ">(Name: \"" + \ diff --git a/bindings/CXX11/adios2/cxx11/Attribute.h b/bindings/CXX11/adios2/cxx11/Attribute.h index 01e80901f4..fd55ab0b5e 100644 --- a/bindings/CXX11/adios2/cxx11/Attribute.h +++ b/bindings/CXX11/adios2/cxx11/Attribute.h @@ -67,6 +67,12 @@ class Attribute */ std::vector Data() const; + /** + * Distinguish single-value attributes from vector attributes + * @return true if single-value, false otherwise + */ + bool IsValue() const; + private: Attribute(core::Attribute *attribute); core::Attribute *m_Attribute = nullptr; diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 43cc0e97aa..404d9206d0 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -168,9 +168,15 @@ if(ADIOS2_HAVE_MPI) core/IOMPI.cpp helper/adiosCommMPI.h helper/adiosCommMPI.cpp helper/adiosMpiHandshake.h helper/adiosMpiHandshake.cpp - engine/ssc/SscReader.cpp engine/ssc/SscReader.tcc - engine/ssc/SscWriter.cpp engine/ssc/SscWriter.tcc engine/ssc/SscHelper.cpp + engine/ssc/SscWriter.cpp + engine/ssc/SscWriterBase.cpp + engine/ssc/SscWriterGeneric.cpp + engine/ssc/SscWriterNaive.cpp + engine/ssc/SscReader.cpp + engine/ssc/SscReaderBase.cpp + engine/ssc/SscReaderGeneric.cpp + engine/ssc/SscReaderNaive.cpp ) set_property(TARGET adios2_core_mpi PROPERTY EXPORT_NAME core_mpi) set_property(TARGET adios2_core_mpi PROPERTY OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_core_mpi) diff --git a/source/adios2/engine/ssc/SscHelper.cpp b/source/adios2/engine/ssc/SscHelper.cpp index 8ae8ad8c15..8fc70fb4f6 100644 --- a/source/adios2/engine/ssc/SscHelper.cpp +++ b/source/adios2/engine/ssc/SscHelper.cpp @@ -9,11 +9,6 @@ */ #include "SscHelper.h" -#include "adios2/common/ADIOSMacros.h" -#include "adios2/helper/adiosFunctions.h" -#include "adios2/helper/adiosType.h" -#include -#include namespace adios2 { @@ -248,55 +243,36 @@ void SerializeAttributes(IO &input, Buffer &output) } } -void Deserialize(const Buffer &input, BlockVecVec &output, IO &io, - const bool regVars, const bool regAttrs) +void DeserializeAttribute(const Buffer &input, uint64_t &pos, IO &io, + const bool regIO) { - for (auto &i : output) - { - i.clear(); - } + const DataType type = static_cast(input[pos]); + ++pos; - uint64_t pos = 2; + uint8_t nameSize = input[pos]; + ++pos; - uint64_t blockSize = input.value(pos); + std::vector namev(nameSize); + std::memcpy(namev.data(), input.data(pos), nameSize); + std::string name = std::string(namev.begin(), namev.end()); + pos += nameSize; + uint64_t size = input.value(pos); pos += 8; - while (pos < blockSize) + if (regIO) { - - uint8_t shapeId = input[pos]; - ++pos; - - if (shapeId == 66) + const auto &attributes = io.GetAttributes(); + auto it = attributes.find(name); + if (it == attributes.end()) { - const DataType type = static_cast(input[pos]); - ++pos; - - uint8_t nameSize = input[pos]; - ++pos; - - std::vector namev(nameSize); - std::memcpy(namev.data(), input.data(pos), nameSize); - std::string name = std::string(namev.begin(), namev.end()); - pos += nameSize; - - uint64_t size = input.value(pos); - pos += 8; - - if (regAttrs) + int rank; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + if (type == DataType::String) { - const auto &attributes = io.GetAttributes(); - auto it = attributes.find(name); - if (it == attributes.end()) - { - int rank; - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - if (type == DataType::String) - { - io.DefineAttribute( - name, std::string(input.data(pos), size)); - } + io.DefineAttribute( + name, std::string(input.data(pos), size)); + } #define declare_type(T) \ else if (type == helper::GetDataType()) \ { \ @@ -309,75 +285,73 @@ void Deserialize(const Buffer &input, BlockVecVec &output, IO &io, io.DefineAttribute(name, input.data(pos), size / sizeof(T)); \ } \ } - ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(declare_type) + ADIOS2_FOREACH_ATTRIBUTE_STDTYPE_1ARG(declare_type) #undef declare_type - else - { - helper::Throw( - "Engine", "SscHelper", "Deserialize", - "unknown attribute data type"); - } - } + else + { + helper::Throw( + "Engine", "SscHelper", "Deserialize", + "unknown attribute data type"); } - pos += size; } - else - { - int rank = input.value(pos); - pos += 4; - output[rank].emplace_back(); - auto &b = output[rank].back(); - b.shapeId = static_cast(shapeId); + } + pos += size; +} - uint8_t nameSize = input[pos]; - ++pos; +void DeserializeVariable(const Buffer &input, const ShapeID shapeId, + uint64_t &pos, BlockInfo &b, IO &io, const bool regIO) +{ + b.shapeId = static_cast(shapeId); - std::vector name(nameSize); - std::memcpy(name.data(), input.data(pos), nameSize); - b.name = std::string(name.begin(), name.end()); - pos += nameSize; + uint8_t nameSize = input[pos]; + ++pos; - b.type = static_cast(input[pos]); - ++pos; + std::vector name(nameSize); + std::memcpy(name.data(), input.data(pos), nameSize); + b.name = std::string(name.begin(), name.end()); + pos += nameSize; - uint8_t shapeSize = input[pos]; - ++pos; - b.shape.resize(shapeSize); - b.start.resize(shapeSize); - b.count.resize(shapeSize); + b.type = static_cast(input[pos]); + ++pos; - std::memcpy(b.shape.data(), input.data(pos), 8 * shapeSize); - pos += 8 * shapeSize; + uint8_t shapeSize = input[pos]; + ++pos; + b.shape.resize(shapeSize); + b.start.resize(shapeSize); + b.count.resize(shapeSize); - std::memcpy(b.start.data(), input.data(pos), 8 * shapeSize); - pos += 8 * shapeSize; + std::memcpy(b.shape.data(), input.data(pos), 8 * shapeSize); + pos += 8 * shapeSize; - std::memcpy(b.count.data(), input.data(pos), 8 * shapeSize); - pos += 8 * shapeSize; + std::memcpy(b.start.data(), input.data(pos), 8 * shapeSize); + pos += 8 * shapeSize; - b.bufferStart = input.value(pos); - pos += 8; + std::memcpy(b.count.data(), input.data(pos), 8 * shapeSize); + pos += 8 * shapeSize; - b.bufferCount = input.value(pos); - pos += 8; + b.bufferStart = input.value(pos); + pos += 8; - uint8_t valueSize = input[pos]; - pos++; - b.value.resize(valueSize); - if (valueSize > 0) - { - std::memcpy(b.value.data(), input.data() + pos, valueSize); - pos += valueSize; - } + b.bufferCount = input.value(pos); + pos += 8; - if (regVars) - { - if (b.type == DataType::None) - { - helper::Throw( - "Engine", "SscHelper", "Deserialize", - "unknown variable data type"); - } + uint8_t valueSize = input[pos]; + pos++; + b.value.resize(valueSize); + if (valueSize > 0) + { + std::memcpy(b.value.data(), input.data() + pos, valueSize); + pos += valueSize; + } + + if (regIO) + { + if (b.type == DataType::None) + { + helper::Throw("Engine", "SscHelper", + "Deserialize", + "unknown variable data type"); + } #define declare_type(T) \ else if (b.type == helper::GetDataType()) \ { \ @@ -409,15 +383,50 @@ void Deserialize(const Buffer &input, BlockVecVec &output, IO &io, } \ } \ } - ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type - else - { - helper::Throw( - "Engine", "SscHelper", "Deserialize", - "unknown variable data type"); - } - } + else + { + helper::Throw("Engine", "SscHelper", + "Deserialize", + "unknown variable data type"); + } + } +} + +void Deserialize(const Buffer &input, BlockVecVec &output, IO &io, + const bool regVars, const bool regAttrs) +{ + for (auto &i : output) + { + i.clear(); + } + + uint64_t pos = 2; + + uint64_t blockSize = input.value(pos); + + pos += 8; + + while (pos < blockSize) + { + + uint8_t shapeId = input[pos]; + ++pos; + + if (shapeId == 66) + { + DeserializeAttribute(input, pos, io, regAttrs); + } + else + { + int rank = input.value(pos); + pos += 4; + output[rank].emplace_back(); + auto &b = output[rank].back(); + + DeserializeVariable(input, static_cast(shapeId), pos, b, + io, regVars); } } } diff --git a/source/adios2/engine/ssc/SscHelper.h b/source/adios2/engine/ssc/SscHelper.h index beefebdb6a..c0c953f1ba 100644 --- a/source/adios2/engine/ssc/SscHelper.h +++ b/source/adios2/engine/ssc/SscHelper.h @@ -11,12 +11,8 @@ #ifndef ADIOS2_ENGINE_SSCHELPER_H_ #define ADIOS2_ENGINE_SSCHELPER_H_ -#include "adios2/common/ADIOSTypes.h" #include "adios2/core/IO.h" -#include "adios2/helper/adiosLog.h" #include -#include -#include namespace adios2 { @@ -152,6 +148,10 @@ RankPosMap CalculateOverlap(BlockVecVec &globalPattern, void SerializeVariables(const BlockVec &input, Buffer &output, const int rank); void SerializeAttributes(IO &input, Buffer &output); +void DeserializeVariable(const Buffer &input, const ShapeID shapeId, + uint64_t &pos, BlockInfo &b, IO &io, const bool regIO); +void DeserializeAttribute(const Buffer &input, uint64_t &pos, IO &io, + const bool regIO); void Deserialize(const Buffer &input, BlockVecVec &output, IO &io, const bool regVars, const bool regAttrs); void AggregateMetadata(const Buffer &localBuffer, Buffer &globalBuffer, diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp index e8461258bc..c1e757a565 100644 --- a/source/adios2/engine/ssc/SscReader.cpp +++ b/source/adios2/engine/ssc/SscReader.cpp @@ -8,10 +8,12 @@ * Author: Jason Wang */ -#include "SscReader.tcc" -#include "adios2/helper/adiosComm.h" +#include "SscReader.h" +#include "SscReaderGeneric.h" +#include "SscReaderNaive.h" #include "adios2/helper/adiosCommMPI.h" -#include "adios2/helper/adiosMpiHandshake.h" +#include "adios2/helper/adiosString.h" +#include namespace adios2 { @@ -27,485 +29,98 @@ SscReader::SscReader(IO &io, const std::string &name, const Mode mode, PERFSTUBS_SCOPED_TIMER_FUNC(); helper::GetParameter(m_IO.m_Parameters, "Verbose", m_Verbosity); - helper::GetParameter(m_IO.m_Parameters, "Threading", m_Threading); - helper::GetParameter(m_IO.m_Parameters, "OpenTimeoutSecs", - m_OpenTimeoutSecs); + helper::GetParameter(m_IO.m_Parameters, "EngineMode", m_EngineMode); - helper::Log("Engine", "SSCReader", "Open", m_Name, 0, m_Comm.Rank(), 5, + helper::Log("Engine", "SscReader", "SscReader", m_EngineMode, + m_Verbosity >= 10 ? m_Comm.Rank() : 0, m_Comm.Rank(), 5, m_Verbosity, helper::LogMode::INFO); - SyncMpiPattern(); -} - -SscReader::~SscReader() { PERFSTUBS_SCOPED_TIMER_FUNC(); } - -void SscReader::BeginStepConsequentFixed() -{ - MPI_Waitall(static_cast(m_MpiRequests.size()), m_MpiRequests.data(), - MPI_STATUS_IGNORE); - m_MpiRequests.clear(); -} - -void SscReader::BeginStepFlexible(StepStatus &status) -{ - m_AllReceivingWriterRanks.clear(); - m_Buffer.resize(1); - m_Buffer[0] = 0; - m_GlobalWritePattern.clear(); - m_GlobalWritePattern.resize(m_StreamSize); - m_LocalReadPattern.clear(); - m_GlobalWritePatternBuffer.clear(); - bool finalStep = SyncWritePattern(); - if (finalStep) - { - status = StepStatus::EndOfStream; - return; - } - MPI_Win_create(NULL, 0, 1, MPI_INFO_NULL, m_StreamComm, &m_MpiWin); -} - -StepStatus SscReader::BeginStep(const StepMode stepMode, - const float timeoutSeconds) -{ - PERFSTUBS_SCOPED_TIMER_FUNC(); - - ++m_CurrentStep; - - helper::Log("Engine", "SSCReader", "BeginStep", - std::to_string(CurrentStep()), 0, m_Comm.Rank(), 5, m_Verbosity, - helper::LogMode::INFO); - - m_StepBegun = true; - - if (m_CurrentStep == 0 || m_WriterDefinitionsLocked == false || - m_ReaderSelectionsLocked == false) - { - if (m_Threading && m_EndStepThread.joinable()) - { - m_EndStepThread.join(); - } - else - { - BeginStepFlexible(m_StepStatus); - } - if (m_StepStatus == StepStatus::EndOfStream) - { - return StepStatus::EndOfStream; - } - } - else - { - BeginStepConsequentFixed(); - } - - for (const auto &r : m_GlobalWritePattern) - { - for (auto &v : r) - { - if (v.shapeId == ShapeID::GlobalValue || - v.shapeId == ShapeID::LocalValue) - { - std::vector value(v.bufferCount); - if (m_CurrentStep == 0 || m_WriterDefinitionsLocked == false || - m_ReaderSelectionsLocked == false) - { - std::memcpy(value.data(), v.value.data(), v.value.size()); - } - else - { - std::memcpy(value.data(), m_Buffer.data() + v.bufferStart, - v.bufferCount); - } - - if (v.type == DataType::String) - { - auto variable = m_IO.InquireVariable(v.name); - if (variable) - { - variable->m_Value = - std::string(value.begin(), value.end()); - variable->m_Min = - std::string(value.begin(), value.end()); - variable->m_Max = - std::string(value.begin(), value.end()); - } - } -#define declare_type(T) \ - else if (v.type == helper::GetDataType()) \ - { \ - auto variable = m_IO.InquireVariable(v.name); \ - if (variable) \ - { \ - std::memcpy(reinterpret_cast(&variable->m_Min), \ - value.data(), value.size()); \ - std::memcpy(reinterpret_cast(&variable->m_Max), \ - value.data(), value.size()); \ - std::memcpy(reinterpret_cast(&variable->m_Value), \ - value.data(), value.size()); \ - } \ - } - ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) -#undef declare_type - else - { - helper::Log("Engine", "SSCReader", "BeginStep", - "unknown data type", 0, m_Comm.Rank(), 0, - m_Verbosity, helper::LogMode::ERROR); - } - } - } - } - - if (m_Buffer[0] == 1) + if (m_EngineMode == "generic") { - return StepStatus::EndOfStream; + m_EngineInstance = std::make_shared( + io, name, mode, CommAsMPI(m_Comm)); } - - return StepStatus::OK; -} - -void SscReader::PerformGets() -{ - - helper::Log("Engine", "SSCReader", "PerformGets", "", 0, m_Comm.Rank(), 5, - m_Verbosity, helper::LogMode::INFO); - - if (m_CurrentStep == 0 || m_WriterDefinitionsLocked == false || - m_ReaderSelectionsLocked == false) + else if (m_EngineMode == "naive") { - ssc::Deserialize(m_GlobalWritePatternBuffer, m_GlobalWritePattern, m_IO, - false, false); - size_t oldSize = m_AllReceivingWriterRanks.size(); - m_AllReceivingWriterRanks = - ssc::CalculateOverlap(m_GlobalWritePattern, m_LocalReadPattern); - CalculatePosition(m_GlobalWritePattern, m_AllReceivingWriterRanks); - size_t newSize = m_AllReceivingWriterRanks.size(); - if (oldSize != newSize) - { - size_t totalDataSize = 0; - for (auto i : m_AllReceivingWriterRanks) - { - totalDataSize += i.second.second; - } - m_Buffer.resize(totalDataSize); - for (const auto &i : m_AllReceivingWriterRanks) - { - MPI_Win_lock(MPI_LOCK_SHARED, i.first, 0, m_MpiWin); - MPI_Get(m_Buffer.data() + i.second.first, - static_cast(i.second.second), MPI_CHAR, i.first, 0, - static_cast(i.second.second), MPI_CHAR, m_MpiWin); - MPI_Win_unlock(i.first, m_MpiWin); - } - } - - for (auto &br : m_LocalReadPattern) - { - if (br.performed) - { - continue; - } - for (const auto &i : m_AllReceivingWriterRanks) - { - const auto &v = m_GlobalWritePattern[i.first]; - for (const auto &b : v) - { - if (b.name == br.name) - { - if (b.type == DataType::String) - { - *reinterpret_cast(br.data) = - std::string(b.value.begin(), b.value.end()); - } -#define declare_type(T) \ - else if (b.type == helper::GetDataType()) \ - { \ - if (b.shapeId == ShapeID::GlobalArray || \ - b.shapeId == ShapeID::LocalArray) \ - { \ - bool empty = false; \ - for (const auto c : b.count) \ - { \ - if (c == 0) \ - { \ - empty = true; \ - } \ - } \ - if (empty) \ - { \ - continue; \ - } \ - helper::NdCopy(m_Buffer.data() + b.bufferStart, b.start, \ - b.count, true, true, \ - reinterpret_cast(br.data), br.start, \ - br.count, true, true, sizeof(T)); \ - } \ - else if (b.shapeId == ShapeID::GlobalValue || \ - b.shapeId == ShapeID::LocalValue) \ - { \ - std::memcpy(br.data, m_Buffer.data() + b.bufferStart, \ - b.bufferCount); \ - } \ + m_EngineInstance = std::make_shared( + io, name, mode, CommAsMPI(m_Comm)); } - ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) -#undef declare_type - else - { - helper::Log("Engine", "SSCReader", "PerformGets", - "unknown data type", 0, m_Comm.Rank(), - 0, m_Verbosity, helper::LogMode::ERROR); - } - } - } - } - br.performed = true; - } - } -} - -size_t SscReader::CurrentStep() const { return m_CurrentStep; } - -void SscReader::EndStepFixed() -{ - if (m_CurrentStep == 0) - { - MPI_Win_free(&m_MpiWin); - SyncReadPattern(); - } - for (const auto &i : m_AllReceivingWriterRanks) - { - m_MpiRequests.emplace_back(); - MPI_Irecv(m_Buffer.data() + i.second.first, - static_cast(i.second.second), MPI_CHAR, i.first, 0, - m_StreamComm, &m_MpiRequests.back()); - } -} - -void SscReader::EndStepFirstFlexible() -{ - MPI_Win_free(&m_MpiWin); - SyncReadPattern(); - BeginStepFlexible(m_StepStatus); } -void SscReader::EndStepConsequentFlexible() -{ - MPI_Win_free(&m_MpiWin); - BeginStepFlexible(m_StepStatus); -} - -void SscReader::EndStep() +StepStatus SscReader::BeginStep(StepMode stepMode, const float timeoutSeconds) { PERFSTUBS_SCOPED_TIMER_FUNC(); - helper::Log("Engine", "SSCReader", "EndStep", std::to_string(CurrentStep()), - 0, m_Comm.Rank(), 5, m_Verbosity, helper::LogMode::INFO); - PerformGets(); + auto ret = m_EngineInstance->BeginStep(stepMode, timeoutSeconds, + m_ReaderSelectionsLocked); - if (m_WriterDefinitionsLocked && m_ReaderSelectionsLocked) - { - EndStepFixed(); - } - else - { - if (m_CurrentStep == 0) - { - if (m_Threading) - { - m_EndStepThread = - std::thread(&SscReader::EndStepFirstFlexible, this); - } - else - { - MPI_Win_free(&m_MpiWin); - SyncReadPattern(); - } - } - else - { - if (m_Threading) - { - m_EndStepThread = - std::thread(&SscReader::EndStepConsequentFlexible, this); - } - else - { - MPI_Win_free(&m_MpiWin); - } - } - } + helper::Log("Engine", "SscReader", "BeginStep", + std::to_string(CurrentStep()), + m_Verbosity >= 10 ? m_Comm.Rank() : 0, m_Comm.Rank(), 5, + m_Verbosity, helper::LogMode::INFO); - m_StepBegun = false; + return ret; } -void SscReader::SyncMpiPattern() +size_t SscReader::CurrentStep() const { - PERFSTUBS_SCOPED_TIMER_FUNC(); - - MPI_Group streamGroup; - MPI_Group readerGroup; - MPI_Comm writerComm; - - helper::HandshakeComm(m_Name, 'r', m_OpenTimeoutSecs, CommAsMPI(m_Comm), - streamGroup, m_WriterGroup, readerGroup, m_StreamComm, - writerComm, m_ReaderComm); - - m_ReaderRank = m_Comm.Rank(); - m_ReaderSize = m_Comm.Size(); - MPI_Comm_rank(m_StreamComm, &m_StreamRank); - MPI_Comm_size(m_StreamComm, &m_StreamSize); - - int writerMasterStreamRank = -1; - MPI_Allreduce(&writerMasterStreamRank, &m_WriterMasterStreamRank, 1, - MPI_INT, MPI_MAX, m_StreamComm); - - int readerMasterStreamRank = -1; - if (m_ReaderRank == 0) - { - readerMasterStreamRank = m_StreamRank; - } - MPI_Allreduce(&readerMasterStreamRank, &m_ReaderMasterStreamRank, 1, - MPI_INT, MPI_MAX, m_StreamComm); + return m_EngineInstance->CurrentStep(); } -bool SscReader::SyncWritePattern() +void SscReader::PerformGets() { PERFSTUBS_SCOPED_TIMER_FUNC(); - - ssc::BroadcastMetadata(m_GlobalWritePatternBuffer, m_WriterMasterStreamRank, - m_StreamComm); - - if (m_GlobalWritePatternBuffer[0] == 1) - { - return true; - } - - m_WriterDefinitionsLocked = m_GlobalWritePatternBuffer[1]; - - ssc::Deserialize(m_GlobalWritePatternBuffer, m_GlobalWritePattern, m_IO, - true, true); - - if (m_Verbosity >= 20 && m_ReaderRank == 0) - { - ssc::PrintBlockVecVec(m_GlobalWritePattern, "Global Write Pattern"); - } - return false; + m_EngineInstance->PerformGets(); } -void SscReader::SyncReadPattern() +void SscReader::EndStep() { PERFSTUBS_SCOPED_TIMER_FUNC(); - ssc::Buffer localBuffer(8); - localBuffer.value() = 8; - - ssc::SerializeVariables(m_LocalReadPattern, localBuffer, m_StreamRank); - - ssc::Buffer globalBuffer; - - ssc::AggregateMetadata(localBuffer, globalBuffer, m_ReaderComm, false, - m_ReaderSelectionsLocked); - - ssc::BroadcastMetadata(globalBuffer, m_ReaderMasterStreamRank, - m_StreamComm); - - ssc::Deserialize(m_GlobalWritePatternBuffer, m_GlobalWritePattern, m_IO, - true, true); - - m_AllReceivingWriterRanks = - ssc::CalculateOverlap(m_GlobalWritePattern, m_LocalReadPattern); - CalculatePosition(m_GlobalWritePattern, m_AllReceivingWriterRanks); - - size_t totalDataSize = 0; - for (auto i : m_AllReceivingWriterRanks) - { - totalDataSize += i.second.second; - } - m_Buffer.resize(totalDataSize); + helper::Log("Engine", "SscReader", "EndStep", std::to_string(CurrentStep()), + m_Verbosity >= 10 ? m_Comm.Rank() : 0, m_Comm.Rank(), 5, + m_Verbosity, helper::LogMode::INFO); - if (m_Verbosity >= 20) - { - for (int i = 0; i < m_ReaderSize; ++i) - { - m_Comm.Barrier(); - if (i == m_ReaderRank) - { - ssc::PrintBlockVec(m_LocalReadPattern, - "\n\nGlobal Read Pattern on Rank " + - std::to_string(m_ReaderRank)); - } - } - m_Comm.Barrier(); - } + return m_EngineInstance->EndStep(m_ReaderSelectionsLocked); } -void SscReader::CalculatePosition(ssc::BlockVecVec &bvv, - ssc::RankPosMap &allRanks) +void SscReader::DoClose(const int transportIndex) { PERFSTUBS_SCOPED_TIMER_FUNC(); - size_t bufferPosition = 0; + helper::Log("Engine", "SscReader", "Close", m_Name, + m_Verbosity >= 10 ? m_Comm.Rank() : 0, m_Comm.Rank(), 5, + m_Verbosity, helper::LogMode::INFO); - for (int rank = 0; rank < static_cast(bvv.size()); ++rank) - { - bool hasOverlap = false; - for (const auto &r : allRanks) - { - if (r.first == rank) - { - hasOverlap = true; - break; - } - } - if (hasOverlap) - { - allRanks[rank].first = bufferPosition; - auto &bv = bvv[rank]; - for (auto &b : bv) - { - b.bufferStart += bufferPosition; - } - size_t currentRankTotalSize = ssc::TotalDataSize(bv); - allRanks[rank].second = currentRankTotalSize + 1; - bufferPosition += currentRankTotalSize + 1; - } - } + m_EngineInstance->Close(transportIndex); } #define declare_type(T) \ void SscReader::DoGetSync(Variable &variable, T *data) \ { \ - helper::Log("Engine", "SSCReader", "GetSync", variable.m_Name, 0, \ - m_Comm.Rank(), 5, m_Verbosity, helper::LogMode::INFO); \ - GetDeferredCommon(variable, data); \ - PerformGets(); \ + PERFSTUBS_SCOPED_TIMER_FUNC(); \ + helper::Log("Engine", "SscReader", "GetSync", variable.m_Name, \ + m_Verbosity >= 10 ? m_Comm.Rank() : 0, m_Comm.Rank(), 5, \ + m_Verbosity, helper::LogMode::INFO); \ + m_EngineInstance->GetDeferred(variable, data); \ + m_EngineInstance->PerformGets(); \ } \ void SscReader::DoGetDeferred(Variable &variable, T *data) \ { \ - helper::Log("Engine", "SSCReader", "GetDeferred", variable.m_Name, 0, \ - m_Comm.Rank(), 5, m_Verbosity, helper::LogMode::INFO); \ - GetDeferredCommon(variable, data); \ + PERFSTUBS_SCOPED_TIMER_FUNC(); \ + helper::Log("Engine", "SscReader", "GetDeferred", variable.m_Name, \ + m_Verbosity >= 10 ? m_Comm.Rank() : 0, m_Comm.Rank(), 5, \ + m_Verbosity, helper::LogMode::INFO); \ + m_EngineInstance->GetDeferred(variable, data); \ } \ std::vector::BPInfo> SscReader::DoBlocksInfo( \ const Variable &variable, const size_t step) const \ { \ - return BlocksInfoCommon(variable, step); \ + return m_EngineInstance->BlocksInfo(variable, step); \ } ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type -void SscReader::DoClose(const int transportIndex) -{ - PERFSTUBS_SCOPED_TIMER_FUNC(); - - helper::Log("Engine", "SSCReader", "Close", m_Name, 0, m_Comm.Rank(), 5, - m_Verbosity, helper::LogMode::INFO); - - if (!m_StepBegun) - { - BeginStep(); - } -} - } // end namespace engine } // end namespace core } // end namespace adios2 diff --git a/source/adios2/engine/ssc/SscReader.h b/source/adios2/engine/ssc/SscReader.h index db4754ef32..3b2ae455e6 100644 --- a/source/adios2/engine/ssc/SscReader.h +++ b/source/adios2/engine/ssc/SscReader.h @@ -11,12 +11,8 @@ #ifndef ADIOS2_ENGINE_SSCREADER_H_ #define ADIOS2_ENGINE_SSCREADER_H_ -#include "SscHelper.h" +#include "SscReaderBase.h" #include "adios2/core/Engine.h" -#include "adios2/helper/adiosMpiHandshake.h" -#include -#include -#include namespace adios2 { @@ -30,7 +26,7 @@ class SscReader : public Engine public: SscReader(IO &adios, const std::string &name, const Mode mode, helper::Comm comm); - virtual ~SscReader(); + ~SscReader() = default; StepStatus BeginStep( StepMode stepMode = StepMode::Read, const float timeoutSeconds = std::numeric_limits::max()) final; @@ -39,39 +35,6 @@ class SscReader : public Engine void EndStep() final; private: - int64_t m_CurrentStep = -1; - bool m_StepBegun = false; - - ssc::BlockVecVec m_GlobalWritePattern; - ssc::BlockVec m_LocalReadPattern; - ssc::Buffer m_GlobalWritePatternBuffer; - - ssc::RankPosMap m_AllReceivingWriterRanks; - ssc::Buffer m_Buffer; - MPI_Win m_MpiWin; - MPI_Group m_WriterGroup; - MPI_Comm m_StreamComm; - MPI_Comm m_ReaderComm; - std::vector m_MpiRequests; - StepStatus m_StepStatus; - std::thread m_EndStepThread; - - int m_StreamRank; - int m_StreamSize; - int m_ReaderRank; - int m_ReaderSize; - int m_WriterMasterStreamRank; - int m_ReaderMasterStreamRank; - - void SyncMpiPattern(); - bool SyncWritePattern(); - void SyncReadPattern(); - void BeginStepConsequentFixed(); - void BeginStepFlexible(StepStatus &status); - void EndStepFixed(); - void EndStepFirstFlexible(); - void EndStepConsequentFlexible(); - #define declare_type(T) \ void DoGetSync(Variable &, T *) final; \ void DoGetDeferred(Variable &, T *) final; \ @@ -80,24 +43,11 @@ class SscReader : public Engine ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type - template - std::vector::BPInfo> - BlocksInfoCommon(const Variable &variable, const size_t step) const; - - void DoClose(const int transportIndex = -1); - - template - void GetDeferredCommon(Variable &variable, T *data); - - template - void GetDeferredDeltaCommon(Variable &variable, T *data); - - void CalculatePosition(ssc::BlockVecVec &mapVec, - ssc::RankPosMap &allOverlapRanks); + void DoClose(const int transportIndex = -1) final; int m_Verbosity = 0; - int m_OpenTimeoutSecs = 10; - bool m_Threading = false; + std::string m_EngineMode = "generic"; + std::shared_ptr m_EngineInstance; }; } // end namespace engine diff --git a/source/adios2/engine/ssc/SscReaderBase.cpp b/source/adios2/engine/ssc/SscReaderBase.cpp new file mode 100644 index 0000000000..2174f9bffc --- /dev/null +++ b/source/adios2/engine/ssc/SscReaderBase.cpp @@ -0,0 +1,69 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * SscReaderBase.cpp + * + * Created on: Mar 3, 2022 + * Author: Jason Wang + */ + +#include "SscReaderBase.h" +#include "adios2/helper/adiosMpiHandshake.h" +#include "adios2/helper/adiosString.h" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ +namespace ssc +{ + +SscReaderBase::SscReaderBase(IO &io, const std::string &name, const Mode mode, + MPI_Comm comm) +: m_Name(name), m_IO(io) +{ + helper::GetParameter(io.m_Parameters, "Verbose", m_Verbosity); + helper::GetParameter(io.m_Parameters, "Threading", m_Threading); + helper::GetParameter(io.m_Parameters, "OpenTimeoutSecs", m_OpenTimeoutSecs); + + SyncMpiPattern(comm); +} + +SscReaderBase::~SscReaderBase() {} + +void SscReaderBase::SyncMpiPattern(MPI_Comm comm) +{ + + MPI_Group streamGroup; + MPI_Group readerGroup; + MPI_Comm writerComm; + + helper::HandshakeComm(m_Name, 'r', m_OpenTimeoutSecs, comm, streamGroup, + m_WriterGroup, readerGroup, m_StreamComm, writerComm, + m_ReaderComm); + + MPI_Comm_rank(comm, &m_ReaderRank); + MPI_Comm_size(comm, &m_ReaderSize); + MPI_Comm_rank(m_StreamComm, &m_StreamRank); + MPI_Comm_size(m_StreamComm, &m_StreamSize); + + int writerMasterStreamRank = -1; + MPI_Allreduce(&writerMasterStreamRank, &m_WriterMasterStreamRank, 1, + MPI_INT, MPI_MAX, m_StreamComm); + + int readerMasterStreamRank = -1; + if (m_ReaderRank == 0) + { + readerMasterStreamRank = m_StreamRank; + } + MPI_Allreduce(&readerMasterStreamRank, &m_ReaderMasterStreamRank, 1, + MPI_INT, MPI_MAX, m_StreamComm); +} + +} +} +} +} diff --git a/source/adios2/engine/ssc/SscReaderBase.h b/source/adios2/engine/ssc/SscReaderBase.h new file mode 100644 index 0000000000..1c381efa8c --- /dev/null +++ b/source/adios2/engine/ssc/SscReaderBase.h @@ -0,0 +1,80 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * SscReaderBase.h + * + * Created on: Mar 3, 2022 + * Author: Jason Wang + */ + +#ifndef ADIOS2_ENGINE_SSCREADERBASE_H_ +#define ADIOS2_ENGINE_SSCREADERBASE_H_ + +#include "SscHelper.h" +#include "adios2/core/IO.h" +#include + +namespace adios2 +{ +namespace core +{ +namespace engine +{ +namespace ssc +{ + +class SscReaderBase +{ + +public: + SscReaderBase(IO &io, const std::string &name, const Mode mode, + MPI_Comm comm); + virtual ~SscReaderBase(); + + virtual StepStatus BeginStep(const StepMode mode, + const float timeoutSeconds, + const bool readerLocked) = 0; + virtual size_t CurrentStep() = 0; + virtual void PerformGets() = 0; + virtual void EndStep(const bool readerLocked) = 0; + virtual void Close(const int transportIndex) = 0; + +#define declare_type(T) \ + virtual void GetDeferred(Variable &, T *) = 0; \ + virtual std::vector::BPInfo> BlocksInfo( \ + const Variable &variable, const size_t step) const = 0; + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + +protected: + void SyncMpiPattern(MPI_Comm comm); + + MPI_Group m_WriterGroup; + MPI_Comm m_StreamComm; + MPI_Comm m_ReaderComm; + + int m_StreamRank; + int m_StreamSize; + int m_ReaderRank; + int m_ReaderSize; + int m_WriterMasterStreamRank; + int m_ReaderMasterStreamRank; + + ssc::Buffer m_Buffer; + + int64_t m_CurrentStep = -1; + std::string m_Name; + int m_Verbosity = 0; + int m_OpenTimeoutSecs = 10; + bool m_Threading = false; + + IO &m_IO; +}; + +} +} +} +} + +#endif // ADIOS2_ENGINE_SSCREADERBASE_H_ diff --git a/source/adios2/engine/ssc/SscReaderGeneric.cpp b/source/adios2/engine/ssc/SscReaderGeneric.cpp new file mode 100644 index 0000000000..694b0b831d --- /dev/null +++ b/source/adios2/engine/ssc/SscReaderGeneric.cpp @@ -0,0 +1,444 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * SscReaderGeneric.cpp + * + * Created on: Mar 3, 2022 + * Author: Jason Wang + */ + +#include "SscReaderGeneric.tcc" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ +namespace ssc +{ + +SscReaderGeneric::SscReaderGeneric(IO &io, const std::string &name, + const Mode mode, MPI_Comm comm) +: SscReaderBase(io, name, mode, comm) +{ +} + +void SscReaderGeneric::BeginStepConsequentFixed() +{ + MPI_Waitall(static_cast(m_MpiRequests.size()), m_MpiRequests.data(), + MPI_STATUS_IGNORE); + m_MpiRequests.clear(); +} + +void SscReaderGeneric::BeginStepFlexible(StepStatus &status) +{ + m_AllReceivingWriterRanks.clear(); + m_Buffer.resize(1); + m_Buffer[0] = 0; + m_GlobalWritePattern.clear(); + m_GlobalWritePattern.resize(m_StreamSize); + m_LocalReadPattern.clear(); + m_GlobalWritePatternBuffer.clear(); + bool finalStep = SyncWritePattern(); + if (finalStep) + { + status = StepStatus::EndOfStream; + return; + } + MPI_Win_create(NULL, 0, 1, MPI_INFO_NULL, m_StreamComm, &m_MpiWin); +} + +StepStatus SscReaderGeneric::BeginStep(const StepMode stepMode, + const float timeoutSeconds, + const bool readerLocked) +{ + + m_ReaderSelectionsLocked = readerLocked; + + ++m_CurrentStep; + + m_StepBegun = true; + + if (m_CurrentStep == 0 || m_WriterDefinitionsLocked == false || + m_ReaderSelectionsLocked == false) + { + if (m_Threading && m_EndStepThread.joinable()) + { + m_EndStepThread.join(); + } + else + { + BeginStepFlexible(m_StepStatus); + } + if (m_StepStatus == StepStatus::EndOfStream) + { + return StepStatus::EndOfStream; + } + } + else + { + BeginStepConsequentFixed(); + } + + for (const auto &r : m_GlobalWritePattern) + { + for (auto &v : r) + { + if (v.shapeId == ShapeID::GlobalValue || + v.shapeId == ShapeID::LocalValue) + { + std::vector value(v.bufferCount); + if (m_CurrentStep == 0 || m_WriterDefinitionsLocked == false || + m_ReaderSelectionsLocked == false) + { + std::memcpy(value.data(), v.value.data(), v.value.size()); + } + else + { + std::memcpy(value.data(), m_Buffer.data() + v.bufferStart, + v.bufferCount); + } + + if (v.type == DataType::String) + { + auto variable = m_IO.InquireVariable(v.name); + if (variable) + { + variable->m_Value = + std::string(value.begin(), value.end()); + variable->m_Min = + std::string(value.begin(), value.end()); + variable->m_Max = + std::string(value.begin(), value.end()); + } + } +#define declare_type(T) \ + else if (v.type == helper::GetDataType()) \ + { \ + auto variable = m_IO.InquireVariable(v.name); \ + if (variable) \ + { \ + std::memcpy(reinterpret_cast(&variable->m_Min), \ + value.data(), value.size()); \ + std::memcpy(reinterpret_cast(&variable->m_Max), \ + value.data(), value.size()); \ + std::memcpy(reinterpret_cast(&variable->m_Value), \ + value.data(), value.size()); \ + } \ + } + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + else + { + helper::Log("Engine", "SscReader", "BeginStep", + "unknown data type", m_ReaderRank, m_ReaderRank, + 0, m_Verbosity, helper::LogMode::ERROR); + } + } + } + } + + if (m_Buffer[0] == 1) + { + return StepStatus::EndOfStream; + } + + return StepStatus::OK; +} + +size_t SscReaderGeneric::CurrentStep() { return m_CurrentStep; } + +void SscReaderGeneric::EndStepFixed() +{ + if (m_CurrentStep == 0) + { + MPI_Win_free(&m_MpiWin); + SyncReadPattern(); + } + for (const auto &i : m_AllReceivingWriterRanks) + { + m_MpiRequests.emplace_back(); + MPI_Irecv(m_Buffer.data() + i.second.first, + static_cast(i.second.second), MPI_CHAR, i.first, 0, + m_StreamComm, &m_MpiRequests.back()); + } +} + +void SscReaderGeneric::EndStepFirstFlexible() +{ + MPI_Win_free(&m_MpiWin); + SyncReadPattern(); + BeginStepFlexible(m_StepStatus); +} + +void SscReaderGeneric::EndStepConsequentFlexible() +{ + MPI_Win_free(&m_MpiWin); + BeginStepFlexible(m_StepStatus); +} + +void SscReaderGeneric::EndStep(const bool readerLocked) +{ + m_ReaderSelectionsLocked = readerLocked; + PerformGets(); + + if (m_WriterDefinitionsLocked && m_ReaderSelectionsLocked) + { + EndStepFixed(); + } + else + { + if (m_CurrentStep == 0) + { + if (m_Threading) + { + m_EndStepThread = + std::thread(&SscReaderGeneric::EndStepFirstFlexible, this); + } + else + { + MPI_Win_free(&m_MpiWin); + SyncReadPattern(); + } + } + else + { + if (m_Threading) + { + m_EndStepThread = std::thread( + &SscReaderGeneric::EndStepConsequentFlexible, this); + } + else + { + MPI_Win_free(&m_MpiWin); + } + } + } + + m_StepBegun = false; +} + +void SscReaderGeneric::PerformGets() +{ + + if (m_CurrentStep == 0 || m_WriterDefinitionsLocked == false || + m_ReaderSelectionsLocked == false) + { + ssc::Deserialize(m_GlobalWritePatternBuffer, m_GlobalWritePattern, m_IO, + false, false); + size_t oldSize = m_AllReceivingWriterRanks.size(); + m_AllReceivingWriterRanks = + ssc::CalculateOverlap(m_GlobalWritePattern, m_LocalReadPattern); + CalculatePosition(m_GlobalWritePattern, m_AllReceivingWriterRanks); + size_t newSize = m_AllReceivingWriterRanks.size(); + if (oldSize != newSize) + { + size_t totalDataSize = 0; + for (auto i : m_AllReceivingWriterRanks) + { + totalDataSize += i.second.second; + } + m_Buffer.resize(totalDataSize); + for (const auto &i : m_AllReceivingWriterRanks) + { + MPI_Win_lock(MPI_LOCK_SHARED, i.first, 0, m_MpiWin); + MPI_Get(m_Buffer.data() + i.second.first, + static_cast(i.second.second), MPI_CHAR, i.first, 0, + static_cast(i.second.second), MPI_CHAR, m_MpiWin); + MPI_Win_unlock(i.first, m_MpiWin); + } + } + + for (auto &br : m_LocalReadPattern) + { + if (br.performed) + { + continue; + } + for (const auto &i : m_AllReceivingWriterRanks) + { + const auto &v = m_GlobalWritePattern[i.first]; + for (const auto &b : v) + { + if (b.name == br.name) + { + if (b.type == DataType::String) + { + *reinterpret_cast(br.data) = + std::string(b.value.begin(), b.value.end()); + } +#define declare_type(T) \ + else if (b.type == helper::GetDataType()) \ + { \ + if (b.shapeId == ShapeID::GlobalArray || \ + b.shapeId == ShapeID::LocalArray) \ + { \ + bool empty = false; \ + for (const auto c : b.count) \ + { \ + if (c == 0) \ + { \ + empty = true; \ + } \ + } \ + if (empty) \ + { \ + continue; \ + } \ + helper::NdCopy(m_Buffer.data() + b.bufferStart, b.start, \ + b.count, true, true, \ + reinterpret_cast(br.data), br.start, \ + br.count, true, true, sizeof(T)); \ + } \ + else if (b.shapeId == ShapeID::GlobalValue || \ + b.shapeId == ShapeID::LocalValue) \ + { \ + std::memcpy(br.data, m_Buffer.data() + b.bufferStart, \ + b.bufferCount); \ + } \ + } + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + else + { + helper::Log("Engine", "SscReader", "PerformGets", + "unknown data type", m_ReaderRank, + m_ReaderRank, 0, m_Verbosity, + helper::LogMode::ERROR); + } + } + } + } + br.performed = true; + } + } +} + +bool SscReaderGeneric::SyncWritePattern() +{ + + ssc::BroadcastMetadata(m_GlobalWritePatternBuffer, m_WriterMasterStreamRank, + m_StreamComm); + + if (m_GlobalWritePatternBuffer[0] == 1) + { + return true; + } + + m_WriterDefinitionsLocked = m_GlobalWritePatternBuffer[1]; + + ssc::Deserialize(m_GlobalWritePatternBuffer, m_GlobalWritePattern, m_IO, + true, true); + + if (m_Verbosity >= 20 && m_ReaderRank == 0) + { + ssc::PrintBlockVecVec(m_GlobalWritePattern, "Global Write Pattern"); + } + return false; +} + +void SscReaderGeneric::SyncReadPattern() +{ + + ssc::Buffer localBuffer(8); + localBuffer.value() = 8; + + ssc::SerializeVariables(m_LocalReadPattern, localBuffer, m_StreamRank); + + ssc::Buffer globalBuffer; + + ssc::AggregateMetadata(localBuffer, globalBuffer, m_ReaderComm, false, + m_ReaderSelectionsLocked); + + ssc::BroadcastMetadata(globalBuffer, m_ReaderMasterStreamRank, + m_StreamComm); + + ssc::Deserialize(m_GlobalWritePatternBuffer, m_GlobalWritePattern, m_IO, + true, true); + + m_AllReceivingWriterRanks = + ssc::CalculateOverlap(m_GlobalWritePattern, m_LocalReadPattern); + CalculatePosition(m_GlobalWritePattern, m_AllReceivingWriterRanks); + + size_t totalDataSize = 0; + for (auto i : m_AllReceivingWriterRanks) + { + totalDataSize += i.second.second; + } + m_Buffer.resize(totalDataSize); + + if (m_Verbosity >= 20) + { + for (int i = 0; i < m_ReaderSize; ++i) + { + MPI_Barrier(m_ReaderComm); + if (i == m_ReaderRank) + { + ssc::PrintBlockVec(m_LocalReadPattern, + "\n\nGlobal Read Pattern on Rank " + + std::to_string(m_ReaderRank)); + } + } + MPI_Barrier(m_ReaderComm); + } +} + +void SscReaderGeneric::CalculatePosition(ssc::BlockVecVec &bvv, + ssc::RankPosMap &allRanks) +{ + + size_t bufferPosition = 0; + + for (int rank = 0; rank < static_cast(bvv.size()); ++rank) + { + bool hasOverlap = false; + for (const auto &r : allRanks) + { + if (r.first == rank) + { + hasOverlap = true; + break; + } + } + if (hasOverlap) + { + allRanks[rank].first = bufferPosition; + auto &bv = bvv[rank]; + for (auto &b : bv) + { + b.bufferStart += bufferPosition; + } + size_t currentRankTotalSize = ssc::TotalDataSize(bv); + allRanks[rank].second = currentRankTotalSize + 1; + bufferPosition += currentRankTotalSize + 1; + } + } +} + +void SscReaderGeneric::Close(const int transportIndex) +{ + if (!m_StepBegun) + { + BeginStep(StepMode::Read, -1.0, m_ReaderSelectionsLocked); + } +} + +#define declare_type(T) \ + void SscReaderGeneric::GetDeferred(Variable &variable, T *data) \ + { \ + GetDeferredCommon(variable, data); \ + } \ + std::vector::BPInfo> SscReaderGeneric::BlocksInfo( \ + const Variable &variable, const size_t step) const \ + { \ + return BlocksInfoCommon(variable, step); \ + } +ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + +} +} +} +} diff --git a/source/adios2/engine/ssc/SscReaderGeneric.h b/source/adios2/engine/ssc/SscReaderGeneric.h new file mode 100644 index 0000000000..d267dc7a14 --- /dev/null +++ b/source/adios2/engine/ssc/SscReaderGeneric.h @@ -0,0 +1,88 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * SscReaderGeneric.h + * + * Created on: Mar 3, 2022 + * Author: Jason Wang + */ + +#ifndef ADIOS2_ENGINE_SSCREADERGENERIC_H_ +#define ADIOS2_ENGINE_SSCREADERGENERIC_H_ + +#include "SscReaderBase.h" +#include "adios2/core/IO.h" +#include + +namespace adios2 +{ +namespace core +{ +namespace engine +{ +namespace ssc +{ + +class SscReaderGeneric : public SscReaderBase +{ + +public: + SscReaderGeneric(IO &io, const std::string &name, const Mode mode, + MPI_Comm comm); + ~SscReaderGeneric() = default; + + StepStatus BeginStep(const StepMode mode, const float timeoutSeconds, + const bool readerLocked) final; + size_t CurrentStep() final; + void PerformGets() final; + void EndStep(const bool readerLocked) final; + void Close(const int transportIndex) final; + +#define declare_type(T) \ + void GetDeferred(Variable &, T *) final; \ + std::vector::BPInfo> BlocksInfo( \ + const Variable &variable, const size_t step) const final; + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + +private: + bool m_StepBegun = false; + bool m_WriterDefinitionsLocked = false; + bool m_ReaderSelectionsLocked = false; + std::thread m_EndStepThread; + StepStatus m_StepStatus; + std::vector m_MpiRequests; + ssc::RankPosMap m_AllReceivingWriterRanks; + ssc::BlockVecVec m_GlobalWritePattern; + ssc::BlockVec m_LocalReadPattern; + ssc::Buffer m_GlobalWritePatternBuffer; + MPI_Win m_MpiWin; + + bool SyncWritePattern(); + void SyncReadPattern(); + void BeginStepConsequentFixed(); + void BeginStepFlexible(StepStatus &status); + void EndStepFixed(); + void EndStepFirstFlexible(); + void EndStepConsequentFlexible(); + void CalculatePosition(ssc::BlockVecVec &mapVec, + ssc::RankPosMap &allOverlapRanks); + + template + std::vector::BPInfo> + BlocksInfoCommon(const Variable &variable, const size_t step) const; + + template + void GetDeferredCommon(Variable &variable, T *data); + + template + void GetDeferredDeltaCommon(Variable &variable, T *data); +}; + +} +} +} +} + +#endif // ADIOS2_ENGINE_SSCREADERGENERIC_H_ diff --git a/source/adios2/engine/ssc/SscReader.tcc b/source/adios2/engine/ssc/SscReaderGeneric.tcc similarity index 83% rename from source/adios2/engine/ssc/SscReader.tcc rename to source/adios2/engine/ssc/SscReaderGeneric.tcc index 6271b69b97..6ab91ab089 100644 --- a/source/adios2/engine/ssc/SscReader.tcc +++ b/source/adios2/engine/ssc/SscReaderGeneric.tcc @@ -2,19 +2,17 @@ * Distributed under the OSI-approved Apache License, Version 2.0. See * accompanying file Copyright.txt for details. * - * SscReader.tcc + * SscReaderGeneric.tcc * - * Created on: Nov 1, 2018 + * Created on: Mar 3, 2022 * Author: Jason Wang */ -#ifndef ADIOS2_ENGINE_SSCREADER_TCC_ -#define ADIOS2_ENGINE_SSCREADER_TCC_ +#ifndef ADIOS2_ENGINE_SSCREADERGENERIC_TCC_ +#define ADIOS2_ENGINE_SSCREADERGENERIC_TCC_ -#include "SscReader.h" -#include "adios2/helper/adiosFunctions.h" -#include -#include +#include "SscReaderGeneric.h" +#include "adios2/helper/adiosMemory.h" namespace adios2 { @@ -22,11 +20,12 @@ namespace core { namespace engine { +namespace ssc +{ template -void SscReader::GetDeferredDeltaCommon(Variable &variable, T *data) +void SscReaderGeneric::GetDeferredDeltaCommon(Variable &variable, T *data) { - PERFSTUBS_SCOPED_TIMER_FUNC(); Dims vStart = variable.m_Start; Dims vCount = variable.m_Count; @@ -64,10 +63,9 @@ void SscReader::GetDeferredDeltaCommon(Variable &variable, T *data) } template <> -void SscReader::GetDeferredCommon(Variable &variable, - std::string *data) +void SscReaderGeneric::GetDeferredCommon(Variable &variable, + std::string *data) { - PERFSTUBS_SCOPED_TIMER_FUNC(); variable.SetData(data); if (m_CurrentStep == 0 || m_WriterDefinitionsLocked == false || @@ -92,9 +90,8 @@ void SscReader::GetDeferredCommon(Variable &variable, } template -void SscReader::GetDeferredCommon(Variable &variable, T *data) +void SscReaderGeneric::GetDeferredCommon(Variable &variable, T *data) { - PERFSTUBS_SCOPED_TIMER_FUNC(); variable.SetData(data); Dims vStart = variable.m_Start; @@ -152,9 +149,10 @@ void SscReader::GetDeferredCommon(Variable &variable, T *data) } else { - helper::Log("Engine", "SSCReader", "GetDeferredCommon", - "unknown ShapeID", 0, m_Comm.Rank(), 0, - m_Verbosity, helper::LogMode::ERROR); + helper::Log("Engine", "SscReader", "GetDeferredCommon", + "unknown ShapeID", m_ReaderRank, + m_ReaderRank, 0, m_Verbosity, + helper::LogMode::ERROR); } } } @@ -164,10 +162,9 @@ void SscReader::GetDeferredCommon(Variable &variable, T *data) template std::vector::BPInfo> -SscReader::BlocksInfoCommon(const Variable &variable, - const size_t step) const +SscReaderGeneric::BlocksInfoCommon(const Variable &variable, + const size_t step) const { - PERFSTUBS_SCOPED_TIMER_FUNC(); std::vector::BPInfo> ret; @@ -215,8 +212,9 @@ SscReader::BlocksInfoCommon(const Variable &variable, return ret; } -} // end namespace engine -} // end namespace core -} // end namespace adios2 +} +} +} +} -#endif // ADIOS2_ENGINE_SSCREADER_TCC_ +#endif // ADIOS2_ENGINE_SSCREADERGENERIC_TCC_ diff --git a/source/adios2/engine/ssc/SscReaderNaive.cpp b/source/adios2/engine/ssc/SscReaderNaive.cpp new file mode 100644 index 0000000000..a52d8fdad4 --- /dev/null +++ b/source/adios2/engine/ssc/SscReaderNaive.cpp @@ -0,0 +1,120 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * SscReaderNaive.cpp + * + * Created on: Mar 7, 2022 + * Author: Jason Wang + */ + +#include "SscReaderNaive.tcc" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ +namespace ssc +{ + +SscReaderNaive::SscReaderNaive(IO &io, const std::string &name, const Mode mode, + MPI_Comm comm) +: SscReaderBase(io, name, mode, comm) +{ +} + +StepStatus SscReaderNaive::BeginStep(const StepMode stepMode, + const float timeoutSeconds, + const bool readerLocked) +{ + + m_Buffer.clear(); + m_BlockMap.clear(); + + ++m_CurrentStep; + + size_t globalSize; + + if (m_ReaderRank == 0) + { + MPI_Recv(&globalSize, 1, MPI_UNSIGNED_LONG_LONG, + m_WriterMasterStreamRank, 0, m_StreamComm, MPI_STATUS_IGNORE); + m_Buffer.resize(globalSize); + // MPI_Recv(m_Buffer.data(), globalSize, MPI_CHAR, + // m_WriterMasterStreamRank, 0, m_StreamComm, MPI_STATUS_IGNORE); + // TODO: revert when the crusher MPI bug is fixed + ssc::Buffer tmp(globalSize); + MPI_Recv(tmp.data(), globalSize, MPI_CHAR, m_WriterMasterStreamRank, 0, + m_StreamComm, MPI_STATUS_IGNORE); + std::memcpy(m_Buffer.data(), tmp.data(), globalSize); + } + + MPI_Bcast(&globalSize, 1, MPI_UNSIGNED_LONG_LONG, 0, m_ReaderComm); + if (m_ReaderRank != 0) + { + m_Buffer.resize(globalSize); + } + if (globalSize == 1) + { + return StepStatus::EndOfStream; + } + MPI_Bcast(m_Buffer.data(), globalSize, MPI_CHAR, 0, m_ReaderComm); + + uint64_t pos = 0; + while (pos < m_Buffer.size()) + { + uint64_t start = pos; + uint64_t end = pos + m_Buffer.value(pos); + pos += m_Buffer.value(pos + 8); + + while (pos < end) + { + uint8_t shapeId = m_Buffer[pos]; + ++pos; + + if (shapeId == 66) + { + DeserializeAttribute(m_Buffer, pos, m_IO, true); + } + else + { + pos += 4; + ssc::BlockInfo b; + DeserializeVariable(m_Buffer, static_cast(shapeId), + pos, b, m_IO, true); + b.bufferStart += start; + m_BlockMap[b.name].push_back(b); + } + } + } + + return StepStatus::OK; +} + +size_t SscReaderNaive::CurrentStep() { return m_CurrentStep; } + +void SscReaderNaive::EndStep(const bool readerLocked) {} + +void SscReaderNaive::PerformGets() {} + +void SscReaderNaive::Close(const int transportIndex) {} + +#define declare_type(T) \ + void SscReaderNaive::GetDeferred(Variable &variable, T *data) \ + { \ + GetDeferredCommon(variable, data); \ + } \ + std::vector::BPInfo> SscReaderNaive::BlocksInfo( \ + const Variable &variable, const size_t step) const \ + { \ + return BlocksInfoCommon(variable, step); \ + } +ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + +} +} +} +} diff --git a/source/adios2/engine/ssc/SscReaderNaive.h b/source/adios2/engine/ssc/SscReaderNaive.h new file mode 100644 index 0000000000..636d327351 --- /dev/null +++ b/source/adios2/engine/ssc/SscReaderNaive.h @@ -0,0 +1,65 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * SscReaderNaive.h + * + * Created on: Mar 7, 2022 + * Author: Jason Wang + */ + +#ifndef ADIOS2_ENGINE_SSCREADERNAIVE_H_ +#define ADIOS2_ENGINE_SSCREADERNAIVE_H_ + +#include "SscReaderBase.h" +#include "adios2/core/IO.h" +#include + +namespace adios2 +{ +namespace core +{ +namespace engine +{ +namespace ssc +{ + +class SscReaderNaive : public SscReaderBase +{ + +public: + SscReaderNaive(IO &io, const std::string &name, const Mode mode, + MPI_Comm comm); + ~SscReaderNaive() = default; + + StepStatus BeginStep(const StepMode mode, const float timeoutSeconds, + const bool readerLocked) final; + size_t CurrentStep() final; + void PerformGets() final; + void EndStep(const bool readerLocked) final; + void Close(const int transportIndex) final; + +#define declare_type(T) \ + void GetDeferred(Variable &, T *) final; \ + std::vector::BPInfo> BlocksInfo( \ + const Variable &variable, const size_t step) const final; + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + +private: + template + std::vector::BPInfo> + BlocksInfoCommon(const Variable &variable, const size_t step) const; + + template + void GetDeferredCommon(Variable &variable, T *data); + + std::unordered_map m_BlockMap; +}; + +} +} +} +} + +#endif // ADIOS2_ENGINE_SSCREADERNAIVE_H_ diff --git a/source/adios2/engine/ssc/SscReaderNaive.tcc b/source/adios2/engine/ssc/SscReaderNaive.tcc new file mode 100644 index 0000000000..52d45c3c42 --- /dev/null +++ b/source/adios2/engine/ssc/SscReaderNaive.tcc @@ -0,0 +1,95 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * SscReaderNaive.tcc + * + * Created on: Mar 7, 2022 + * Author: Jason Wang + */ + +#ifndef ADIOS2_ENGINE_SSCREADERNAIVE_TCC_ +#define ADIOS2_ENGINE_SSCREADERNAIVE_TCC_ + +#include "SscReaderNaive.h" +#include "adios2/helper/adiosMemory.h" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ +namespace ssc +{ + +template <> +void SscReaderNaive::GetDeferredCommon(Variable &variable, + std::string *data) +{ + variable.SetData(data); + + for (const auto &b : m_BlockMap[variable.m_Name]) + { + if (b.name == variable.m_Name) + { + *data = + std::string(m_Buffer.data() + b.bufferStart, + m_Buffer.data() + b.bufferStart + b.bufferCount); + variable.m_Value = *data; + variable.m_Min = *data; + variable.m_Max = *data; + } + } +} + +template +void SscReaderNaive::GetDeferredCommon(Variable &variable, T *data) +{ + variable.SetData(data); + + Dims vStart = variable.m_Start; + Dims vCount = variable.m_Count; + Dims vShape = variable.m_Shape; + + if (m_IO.m_ArrayOrder != ArrayOrdering::RowMajor) + { + std::reverse(vStart.begin(), vStart.end()); + std::reverse(vCount.begin(), vCount.end()); + std::reverse(vShape.begin(), vShape.end()); + } + + for (const auto &b : m_BlockMap[variable.m_Name]) + { + if (b.shapeId == ShapeID::GlobalArray || + b.shapeId == ShapeID::LocalArray) + { + helper::NdCopy(m_Buffer.data() + b.bufferStart, b.start, + b.count, true, true, reinterpret_cast(data), + vStart, vCount, true, true, sizeof(T)); + } + else if (b.shapeId == ShapeID::GlobalValue || + b.shapeId == ShapeID::LocalValue) + { + std::memcpy(data, m_Buffer.data() + b.bufferStart, b.bufferCount); + } + } +} + +template +std::vector::BPInfo> +SscReaderNaive::BlocksInfoCommon(const Variable &variable, + const size_t step) const +{ + + std::vector::BPInfo> ret; + + return ret; +} + +} +} +} +} + +#endif // ADIOS2_ENGINE_SSCREADERNAIVE_TCC_ diff --git a/source/adios2/engine/ssc/SscWriter.cpp b/source/adios2/engine/ssc/SscWriter.cpp index a51cd53f3d..973ecfbe59 100644 --- a/source/adios2/engine/ssc/SscWriter.cpp +++ b/source/adios2/engine/ssc/SscWriter.cpp @@ -8,8 +8,12 @@ * Author: Jason Wang */ -#include "SscWriter.tcc" +#include "SscWriter.h" +#include "SscWriterGeneric.h" +#include "SscWriterNaive.h" #include "adios2/helper/adiosCommMPI.h" +#include "adios2/helper/adiosString.h" +#include namespace adios2 { @@ -24,351 +28,95 @@ SscWriter::SscWriter(IO &io, const std::string &name, const Mode mode, { PERFSTUBS_SCOPED_TIMER_FUNC(); + helper::GetParameter(m_IO.m_Parameters, "EngineMode", m_EngineMode); helper::GetParameter(m_IO.m_Parameters, "Verbose", m_Verbosity); - helper::GetParameter(m_IO.m_Parameters, "Threading", m_Threading); - helper::GetParameter(m_IO.m_Parameters, "OpenTimeoutSecs", - m_OpenTimeoutSecs); - helper::Log("Engine", "SSCWriter", "Open", m_Name, 0, m_Comm.Rank(), 5, + helper::Log("Engine", "SscWriter", "SscWriter", m_EngineMode, + m_Verbosity >= 10 ? m_Comm.Rank() : 0, m_Comm.Rank(), 5, m_Verbosity, helper::LogMode::INFO); - int providedMpiMode; - MPI_Query_thread(&providedMpiMode); - if (m_Threading && providedMpiMode != MPI_THREAD_MULTIPLE) + if (m_EngineMode == "generic") { - m_Threading = false; - helper::Log("Engine", "SSCWriter", "Open", - "SSC Threading disabled as MPI is not initialized with " - "multi-threads", - 0, m_Comm.Rank(), 1, m_Verbosity, helper::LogMode::WARNING); + m_EngineInstance = std::make_shared( + io, name, mode, CommAsMPI(m_Comm)); + } + else if (m_EngineMode == "naive") + { + m_EngineInstance = std::make_shared( + io, name, mode, CommAsMPI(m_Comm)); } - - SyncMpiPattern(); } StepStatus SscWriter::BeginStep(StepMode mode, const float timeoutSeconds) { PERFSTUBS_SCOPED_TIMER_FUNC(); - if (m_Threading && m_EndStepThread.joinable()) - { - m_EndStepThread.join(); - } - - ++m_CurrentStep; - - helper::Log("Engine", "SSCWriter", "BeginStep", - std::to_string(CurrentStep()), 0, m_Comm.Rank(), 5, m_Verbosity, - helper::LogMode::INFO); - - if (m_CurrentStep == 0 || m_WriterDefinitionsLocked == false || - m_ReaderSelectionsLocked == false) - { - m_Buffer.resize(1); - m_Buffer[0] = 0; - m_GlobalWritePattern.clear(); - m_GlobalWritePattern.resize(m_StreamSize); - m_GlobalReadPattern.clear(); - m_GlobalReadPattern.resize(m_StreamSize); - } - - if (m_CurrentStep > 1) - { - if (m_WriterDefinitionsLocked && m_ReaderSelectionsLocked) - { - MPI_Waitall(static_cast(m_MpiRequests.size()), - m_MpiRequests.data(), MPI_STATUSES_IGNORE); - m_MpiRequests.clear(); - } - else - { - MPI_Win_free(&m_MpiWin); - } - } - - return StepStatus::OK; -} - -size_t SscWriter::CurrentStep() const { return m_CurrentStep; } + auto ret = m_EngineInstance->BeginStep(mode, timeoutSeconds, + m_WriterDefinitionsLocked); -void SscWriter::PerformPuts() -{ - PERFSTUBS_SCOPED_TIMER_FUNC(); - helper::Log("Engine", "SSCWriter", "PerformPuts", "", 0, m_Comm.Rank(), 5, + helper::Log("Engine", "SscWriter", "BeginStep", + std::to_string(CurrentStep()), + m_Verbosity >= 10 ? m_Comm.Rank() : 0, m_Comm.Rank(), 5, m_Verbosity, helper::LogMode::INFO); -} - -void SscWriter::EndStepFirst() -{ - PERFSTUBS_SCOPED_TIMER_FUNC(); - SyncWritePattern(); - MPI_Win_create(m_Buffer.data(), m_Buffer.size(), 1, MPI_INFO_NULL, - m_StreamComm, &m_MpiWin); - MPI_Win_free(&m_MpiWin); - SyncReadPattern(); + return ret; } -void SscWriter::EndStepConsequentFixed() +size_t SscWriter::CurrentStep() const { - PERFSTUBS_SCOPED_TIMER_FUNC(); - for (const auto &i : m_AllSendingReaderRanks) - { - m_MpiRequests.emplace_back(); - MPI_Isend(m_Buffer.data(), static_cast(m_Buffer.size()), MPI_CHAR, - i.first, 0, m_StreamComm, &m_MpiRequests.back()); - } + return m_EngineInstance->CurrentStep(); } -void SscWriter::EndStepConsequentFlexible() +void SscWriter::PerformPuts() { PERFSTUBS_SCOPED_TIMER_FUNC(); - SyncWritePattern(); - MPI_Win_create(m_Buffer.data(), m_Buffer.size(), 1, MPI_INFO_NULL, - m_StreamComm, &m_MpiWin); + m_EngineInstance->PerformPuts(); } void SscWriter::EndStep() { PERFSTUBS_SCOPED_TIMER_FUNC(); - helper::Log("Engine", "SSCWriter", "EndStep", std::to_string(CurrentStep()), - 0, m_Comm.Rank(), 5, m_Verbosity, helper::LogMode::INFO); - - if (m_CurrentStep == 0) - { - if (m_Threading) - { - m_EndStepThread = std::thread(&SscWriter::EndStepFirst, this); - } - else - { - EndStepFirst(); - } - } - else - { - if (m_WriterDefinitionsLocked && m_ReaderSelectionsLocked) - { - EndStepConsequentFixed(); - } - else - { - if (m_Threading) - { - m_EndStepThread = - std::thread(&SscWriter::EndStepConsequentFlexible, this); - } - else - { - EndStepConsequentFlexible(); - } - } - } -} - -void SscWriter::Flush(const int transportIndex) -{ - PERFSTUBS_SCOPED_TIMER_FUNC(); -} - -void SscWriter::SyncMpiPattern() -{ - PERFSTUBS_SCOPED_TIMER_FUNC(); - - helper::Log("Engine", "SSCWriter", "SyncMpiPattern", "", 0, m_Comm.Rank(), - 5, m_Verbosity, helper::LogMode::INFO); - - MPI_Group streamGroup; - MPI_Group writerGroup; - MPI_Comm readerComm; - - helper::HandshakeComm(m_Name, 'w', m_OpenTimeoutSecs, CommAsMPI(m_Comm), - streamGroup, writerGroup, m_ReaderGroup, m_StreamComm, - m_WriterComm, readerComm, m_Verbosity); - - m_WriterRank = m_Comm.Rank(); - m_WriterSize = m_Comm.Size(); - MPI_Comm_rank(m_StreamComm, &m_StreamRank); - MPI_Comm_size(m_StreamComm, &m_StreamSize); - - int writerMasterStreamRank = -1; - if (m_WriterRank == 0) - { - writerMasterStreamRank = m_StreamRank; - } - MPI_Allreduce(&writerMasterStreamRank, &m_WriterMasterStreamRank, 1, - MPI_INT, MPI_MAX, m_StreamComm); - - int readerMasterStreamRank = -1; - MPI_Allreduce(&readerMasterStreamRank, &m_ReaderMasterStreamRank, 1, - MPI_INT, MPI_MAX, m_StreamComm); -} - -void SscWriter::SyncWritePattern(bool finalStep) -{ - PERFSTUBS_SCOPED_TIMER_FUNC(); - - helper::Log("Engine", "SSCWriter", "SyncWritePattern", "", 0, m_Comm.Rank(), - 5, m_Verbosity, helper::LogMode::INFO); - - ssc::Buffer localBuffer(8); - localBuffer.value() = 8; - - ssc::SerializeVariables(m_GlobalWritePattern[m_StreamRank], localBuffer, - m_StreamRank); - - if (m_WriterRank == 0) - { - ssc::SerializeAttributes(m_IO, localBuffer); - } - - ssc::Buffer globalBuffer; - - ssc::AggregateMetadata(localBuffer, globalBuffer, m_WriterComm, finalStep, - m_WriterDefinitionsLocked); - - ssc::BroadcastMetadata(globalBuffer, m_WriterMasterStreamRank, - m_StreamComm); - - ssc::Deserialize(globalBuffer, m_GlobalWritePattern, m_IO, false, false); + helper::Log("Engine", "SscWriter", "EndStep", std::to_string(CurrentStep()), + m_Verbosity >= 10 ? m_Comm.Rank() : 0, m_Comm.Rank(), 5, + m_Verbosity, helper::LogMode::INFO); - if (m_Verbosity >= 20 && m_WriterRank == 0) - { - ssc::PrintBlockVecVec(m_GlobalWritePattern, "Global Write Pattern"); - } + m_EngineInstance->EndStep(m_WriterDefinitionsLocked); } -void SscWriter::SyncReadPattern() +void SscWriter::DoClose(const int transportIndex) { PERFSTUBS_SCOPED_TIMER_FUNC(); - helper::Log("Engine", "SSCWriter", "SyncReadPattern", "", 0, m_Comm.Rank(), - 5, m_Verbosity, helper::LogMode::INFO); - - ssc::Buffer globalBuffer; - - ssc::BroadcastMetadata(globalBuffer, m_ReaderMasterStreamRank, - m_StreamComm); - - m_ReaderSelectionsLocked = globalBuffer[1]; - - ssc::Deserialize(globalBuffer, m_GlobalReadPattern, m_IO, false, false); - m_AllSendingReaderRanks = ssc::CalculateOverlap( - m_GlobalReadPattern, m_GlobalWritePattern[m_StreamRank]); - CalculatePosition(m_GlobalWritePattern, m_GlobalReadPattern, m_WriterRank, - m_AllSendingReaderRanks); - - if (m_Verbosity >= 10) - { - for (int i = 0; i < m_WriterSize; ++i) - { - m_Comm.Barrier(); - if (i == m_WriterRank) - { - ssc::PrintRankPosMap(m_AllSendingReaderRanks, - "Rank Pos Map for Writer " + - std::to_string(m_WriterRank)); - } - } - m_Comm.Barrier(); - } -} + helper::Log("Engine", "SscWriter", "Close", m_Name, + m_Verbosity >= 10 ? m_Comm.Rank() : 0, m_Comm.Rank(), 5, + m_Verbosity, helper::LogMode::INFO); -void SscWriter::CalculatePosition(ssc::BlockVecVec &writerVecVec, - ssc::BlockVecVec &readerVecVec, - const int writerRank, - ssc::RankPosMap &allOverlapRanks) -{ - PERFSTUBS_SCOPED_TIMER_FUNC(); - for (auto &overlapRank : allOverlapRanks) - { - auto &readerRankMap = readerVecVec[overlapRank.first]; - auto currentReaderOverlapWriterRanks = - CalculateOverlap(writerVecVec, readerRankMap); - size_t bufferPosition = 0; - for (int rank = 0; rank < static_cast(writerVecVec.size()); ++rank) - { - bool hasOverlap = false; - for (const auto &r : currentReaderOverlapWriterRanks) - { - if (r.first == rank) - { - hasOverlap = true; - break; - } - } - if (hasOverlap) - { - currentReaderOverlapWriterRanks[rank].first = bufferPosition; - auto &bv = writerVecVec[rank]; - size_t currentRankTotalSize = TotalDataSize(bv) + 1; - currentReaderOverlapWriterRanks[rank].second = - currentRankTotalSize; - bufferPosition += currentRankTotalSize; - } - } - allOverlapRanks[overlapRank.first] = - currentReaderOverlapWriterRanks[writerRank]; - } + m_EngineInstance->Close(transportIndex); } #define declare_type(T) \ void SscWriter::DoPutSync(Variable &variable, const T *data) \ { \ - helper::Log("Engine", "SSCWriter", "PutSync", variable.m_Name, 0, \ - m_Comm.Rank(), 5, m_Verbosity, helper::LogMode::INFO); \ - PutDeferredCommon(variable, data); \ - PerformPuts(); \ + PERFSTUBS_SCOPED_TIMER_FUNC(); \ + helper::Log("Engine", "SscWriter", "PutSync", variable.m_Name, \ + m_Verbosity >= 10 ? m_Comm.Rank() : 0, m_Comm.Rank(), 5, \ + m_Verbosity, helper::LogMode::INFO); \ + m_EngineInstance->PutDeferred(variable, data); \ + m_EngineInstance->PerformPuts(); \ } \ void SscWriter::DoPutDeferred(Variable &variable, const T *data) \ { \ - helper::Log("Engine", "SSCWriter", "PutDeferred", variable.m_Name, 0, \ - m_Comm.Rank(), 5, m_Verbosity, helper::LogMode::INFO); \ - PutDeferredCommon(variable, data); \ + PERFSTUBS_SCOPED_TIMER_FUNC(); \ + helper::Log("Engine", "SscWriter", "PutDeferred", variable.m_Name, \ + m_Verbosity >= 10 ? m_Comm.Rank() : 0, m_Comm.Rank(), 5, \ + m_Verbosity, helper::LogMode::INFO); \ + m_EngineInstance->PutDeferred(variable, data); \ } ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type -void SscWriter::DoClose(const int transportIndex) -{ - PERFSTUBS_SCOPED_TIMER_FUNC(); - - helper::Log("Engine", "SSCWriter", "Close", m_Name, 0, m_Comm.Rank(), 5, - m_Verbosity, helper::LogMode::INFO); - - if (m_Threading && m_EndStepThread.joinable()) - { - m_EndStepThread.join(); - } - - if (m_WriterDefinitionsLocked && m_ReaderSelectionsLocked) - { - if (m_CurrentStep > 0) - { - MPI_Waitall(static_cast(m_MpiRequests.size()), - m_MpiRequests.data(), MPI_STATUSES_IGNORE); - m_MpiRequests.clear(); - } - - m_Buffer[0] = 1; - - std::vector requests; - for (const auto &i : m_AllSendingReaderRanks) - { - requests.emplace_back(); - MPI_Isend(m_Buffer.data(), 1, MPI_CHAR, i.first, 0, m_StreamComm, - &requests.back()); - } - MPI_Waitall(static_cast(requests.size()), requests.data(), - MPI_STATUS_IGNORE); - } - else - { - MPI_Win_free(&m_MpiWin); - SyncWritePattern(true); - } -} +void SscWriter::Flush(const int transportIndex) {} } // end namespace engine } // end namespace core diff --git a/source/adios2/engine/ssc/SscWriter.h b/source/adios2/engine/ssc/SscWriter.h index cd0aadd371..df0f34541a 100644 --- a/source/adios2/engine/ssc/SscWriter.h +++ b/source/adios2/engine/ssc/SscWriter.h @@ -11,10 +11,8 @@ #ifndef ADIOS2_ENGINE_SSCWRITER_H_ #define ADIOS2_ENGINE_SSCWRITER_H_ -#include "SscHelper.h" +#include "SscWriterBase.h" #include "adios2/core/Engine.h" -#include "adios2/helper/adiosMpiHandshake.h" -#include namespace adios2 { @@ -27,7 +25,7 @@ class SscWriter : public Engine { public: - SscWriter(IO &adios, const std::string &name, const Mode mode, + SscWriter(IO &io, const std::string &name, const Mode mode, helper::Comm comm); ~SscWriter() = default; @@ -40,34 +38,6 @@ class SscWriter : public Engine void Flush(const int transportIndex = -1) final; private: - int64_t m_CurrentStep = -1; - - ssc::BlockVecVec m_GlobalWritePattern; - ssc::BlockVecVec m_GlobalReadPattern; - - ssc::RankPosMap m_AllSendingReaderRanks; - ssc::Buffer m_Buffer; - MPI_Win m_MpiWin; - MPI_Group m_ReaderGroup; - MPI_Comm m_StreamComm; - MPI_Comm m_WriterComm; - std::vector m_MpiRequests; - std::thread m_EndStepThread; - - int m_StreamRank; - int m_StreamSize; - int m_WriterRank; - int m_WriterSize; - int m_WriterMasterStreamRank; - int m_ReaderMasterStreamRank; - - void SyncMpiPattern(); - void SyncWritePattern(bool finalStep = false); - void SyncReadPattern(); - void EndStepFirst(); - void EndStepConsequentFixed(); - void EndStepConsequentFlexible(); - #define declare_type(T) \ void DoPutSync(Variable &, const T *) final; \ void DoPutDeferred(Variable &, const T *) final; @@ -76,16 +46,9 @@ class SscWriter : public Engine void DoClose(const int transportIndex = -1) final; - template - void PutDeferredCommon(Variable &variable, const T *values); - - void CalculatePosition(ssc::BlockVecVec &writerMapVec, - ssc::BlockVecVec &readerMapVec, const int writerRank, - ssc::RankPosMap &allOverlapRanks); - int m_Verbosity = 0; - int m_OpenTimeoutSecs = 10; - bool m_Threading = false; + std::string m_EngineMode = "generic"; + std::shared_ptr m_EngineInstance; }; } // end namespace engine diff --git a/source/adios2/engine/ssc/SscWriterBase.cpp b/source/adios2/engine/ssc/SscWriterBase.cpp new file mode 100644 index 0000000000..f1c2a8e5b2 --- /dev/null +++ b/source/adios2/engine/ssc/SscWriterBase.cpp @@ -0,0 +1,77 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * SscWriterBase.cpp + * + * Created on: Mar 3, 2022 + * Author: Jason Wang + */ + +#include "SscWriterBase.h" +#include "adios2/helper/adiosMpiHandshake.h" +#include "adios2/helper/adiosString.h" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ +namespace ssc +{ + +SscWriterBase::SscWriterBase(IO &io, const std::string &name, const Mode mode, + MPI_Comm comm) +: m_Name(name), m_IO(io) +{ + + helper::GetParameter(io.m_Parameters, "Verbose", m_Verbosity); + helper::GetParameter(io.m_Parameters, "Threading", m_Threading); + helper::GetParameter(io.m_Parameters, "OpenTimeoutSecs", m_OpenTimeoutSecs); + + int providedMpiMode; + MPI_Query_thread(&providedMpiMode); + if (m_Threading && providedMpiMode != MPI_THREAD_MULTIPLE) + { + m_Threading = false; + } + + SyncMpiPattern(comm); +} + +SscWriterBase::~SscWriterBase() {} + +void SscWriterBase::SyncMpiPattern(MPI_Comm comm) +{ + MPI_Group streamGroup; + MPI_Group writerGroup; + MPI_Comm readerComm; + + helper::HandshakeComm(m_Name, 'w', m_OpenTimeoutSecs, comm, streamGroup, + writerGroup, m_ReaderGroup, m_StreamComm, + m_WriterComm, readerComm, m_Verbosity); + + MPI_Comm_rank(comm, &m_WriterRank); + MPI_Comm_size(comm, &m_WriterSize); + + MPI_Comm_rank(m_StreamComm, &m_StreamRank); + MPI_Comm_size(m_StreamComm, &m_StreamSize); + + int writerMasterStreamRank = -1; + if (m_WriterRank == 0) + { + writerMasterStreamRank = m_StreamRank; + } + MPI_Allreduce(&writerMasterStreamRank, &m_WriterMasterStreamRank, 1, + MPI_INT, MPI_MAX, m_StreamComm); + + int readerMasterStreamRank = -1; + MPI_Allreduce(&readerMasterStreamRank, &m_ReaderMasterStreamRank, 1, + MPI_INT, MPI_MAX, m_StreamComm); +} + +} +} +} +} diff --git a/source/adios2/engine/ssc/SscWriterBase.h b/source/adios2/engine/ssc/SscWriterBase.h new file mode 100644 index 0000000000..d6baf1bab9 --- /dev/null +++ b/source/adios2/engine/ssc/SscWriterBase.h @@ -0,0 +1,77 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * SscWriterBase.h + * + * Created on: Mar 3, 2022 + * Author: Jason Wang + */ + +#ifndef ADIOS2_ENGINE_SSCWRITERBASE_H_ +#define ADIOS2_ENGINE_SSCWRITERBASE_H_ + +#include "SscHelper.h" +#include "adios2/core/IO.h" +#include + +namespace adios2 +{ +namespace core +{ +namespace engine +{ +namespace ssc +{ + +class SscWriterBase +{ + +public: + SscWriterBase(IO &io, const std::string &name, const Mode mode, + MPI_Comm comm); + virtual ~SscWriterBase(); + + virtual StepStatus BeginStep(const StepMode mode, + const float timeoutSeconds, + const bool writerLocked) = 0; + virtual size_t CurrentStep() = 0; + virtual void PerformPuts() = 0; + virtual void EndStep(const bool writerLocked) = 0; + virtual void Close(const int transportIndex) = 0; + +#define declare_type(T) virtual void PutDeferred(Variable &, const T *) = 0; + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + +protected: + void SyncMpiPattern(MPI_Comm comm); + + MPI_Group m_ReaderGroup; + MPI_Comm m_StreamComm; + MPI_Comm m_WriterComm; + + int m_StreamRank; + int m_StreamSize; + int m_WriterRank; + int m_WriterSize; + int m_WriterMasterStreamRank; + int m_ReaderMasterStreamRank; + + ssc::Buffer m_Buffer; + + int64_t m_CurrentStep = -1; + std::string m_Name; + int m_Verbosity = 0; + int m_OpenTimeoutSecs = 10; + bool m_Threading = false; + + IO &m_IO; +}; + +} +} +} +} + +#endif // ADIOS2_ENGINE_SSCWRITERBASE_H_ diff --git a/source/adios2/engine/ssc/SscWriterGeneric.cpp b/source/adios2/engine/ssc/SscWriterGeneric.cpp new file mode 100644 index 0000000000..2a894dcefb --- /dev/null +++ b/source/adios2/engine/ssc/SscWriterGeneric.cpp @@ -0,0 +1,291 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * SscWriterGeneric.cpp + * + * Created on: Mar 3, 2022 + * Author: Jason Wang + */ + +#include "SscWriterGeneric.tcc" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ +namespace ssc +{ + +SscWriterGeneric::SscWriterGeneric(IO &io, const std::string &name, + const Mode mode, MPI_Comm comm) +: SscWriterBase(io, name, mode, comm) +{ +} + +StepStatus SscWriterGeneric::BeginStep(const StepMode mode, + const float timeoutSeconds, + const bool writerLocked) +{ + + m_WriterDefinitionsLocked = writerLocked; + + if (m_Threading && m_EndStepThread.joinable()) + { + m_EndStepThread.join(); + } + + ++m_CurrentStep; + + if (m_CurrentStep == 0 || m_WriterDefinitionsLocked == false || + m_ReaderSelectionsLocked == false) + { + m_Buffer.resize(1); + m_Buffer[0] = 0; + m_GlobalWritePattern.clear(); + m_GlobalWritePattern.resize(m_StreamSize); + m_GlobalReadPattern.clear(); + m_GlobalReadPattern.resize(m_StreamSize); + } + + if (m_CurrentStep > 1) + { + if (m_WriterDefinitionsLocked && m_ReaderSelectionsLocked) + { + MPI_Waitall(static_cast(m_MpiRequests.size()), + m_MpiRequests.data(), MPI_STATUSES_IGNORE); + m_MpiRequests.clear(); + } + else + { + MPI_Win_free(&m_MpiWin); + } + } + + return StepStatus::OK; +} + +size_t SscWriterGeneric::CurrentStep() { return m_CurrentStep; } + +void SscWriterGeneric::PerformPuts() {} + +void SscWriterGeneric::EndStep(const bool writerLocked) +{ + + m_WriterDefinitionsLocked = writerLocked; + + if (m_CurrentStep == 0) + { + if (m_Threading) + { + m_EndStepThread = + std::thread(&SscWriterGeneric::EndStepFirst, this); + } + else + { + EndStepFirst(); + } + } + else + { + if (m_WriterDefinitionsLocked && m_ReaderSelectionsLocked) + { + EndStepConsequentFixed(); + } + else + { + if (m_Threading) + { + m_EndStepThread = std::thread( + &SscWriterGeneric::EndStepConsequentFlexible, this); + } + else + { + EndStepConsequentFlexible(); + } + } + } +} + +void SscWriterGeneric::Close(const int transportIndex) +{ + if (m_Threading && m_EndStepThread.joinable()) + { + m_EndStepThread.join(); + } + + if (m_WriterDefinitionsLocked && m_ReaderSelectionsLocked) + { + if (m_CurrentStep > 0) + { + MPI_Waitall(static_cast(m_MpiRequests.size()), + m_MpiRequests.data(), MPI_STATUSES_IGNORE); + m_MpiRequests.clear(); + } + + m_Buffer[0] = 1; + + std::vector requests; + for (const auto &i : m_AllSendingReaderRanks) + { + requests.emplace_back(); + MPI_Isend(m_Buffer.data(), 1, MPI_CHAR, i.first, 0, m_StreamComm, + &requests.back()); + } + MPI_Waitall(static_cast(requests.size()), requests.data(), + MPI_STATUS_IGNORE); + } + else + { + MPI_Win_free(&m_MpiWin); + SyncWritePattern(true); + } +} + +#define declare_type(T) \ + void SscWriterGeneric::PutDeferred(Variable &variable, const T *data) \ + { \ + PutDeferredCommon(variable, data); \ + } +ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + +void SscWriterGeneric::EndStepFirst() +{ + SyncWritePattern(); + MPI_Win_create(m_Buffer.data(), m_Buffer.size(), 1, MPI_INFO_NULL, + m_StreamComm, &m_MpiWin); + MPI_Win_free(&m_MpiWin); + SyncReadPattern(); +} + +void SscWriterGeneric::EndStepConsequentFixed() +{ + for (const auto &i : m_AllSendingReaderRanks) + { + m_MpiRequests.emplace_back(); + MPI_Isend(m_Buffer.data(), static_cast(m_Buffer.size()), MPI_CHAR, + i.first, 0, m_StreamComm, &m_MpiRequests.back()); + } +} + +void SscWriterGeneric::EndStepConsequentFlexible() +{ + SyncWritePattern(); + MPI_Win_create(m_Buffer.data(), m_Buffer.size(), 1, MPI_INFO_NULL, + m_StreamComm, &m_MpiWin); +} + +void SscWriterGeneric::SyncWritePattern(bool finalStep) +{ + + helper::Log("Engine", "SscWriter", "SyncWritePattern", "", + m_Verbosity >= 10 ? m_WriterRank : 0, m_WriterRank, 5, + m_Verbosity, helper::LogMode::INFO); + + ssc::Buffer localBuffer(8); + localBuffer.value() = 8; + + ssc::SerializeVariables(m_GlobalWritePattern[m_StreamRank], localBuffer, + m_StreamRank); + + if (m_WriterRank == 0) + { + ssc::SerializeAttributes(m_IO, localBuffer); + } + + ssc::Buffer globalBuffer; + + ssc::AggregateMetadata(localBuffer, globalBuffer, m_WriterComm, finalStep, + m_WriterDefinitionsLocked); + + ssc::BroadcastMetadata(globalBuffer, m_WriterMasterStreamRank, + m_StreamComm); + + ssc::Deserialize(globalBuffer, m_GlobalWritePattern, m_IO, false, false); + + if (m_Verbosity >= 20 && m_WriterRank == 0) + { + ssc::PrintBlockVecVec(m_GlobalWritePattern, "Global Write Pattern"); + } +} + +void SscWriterGeneric::SyncReadPattern() +{ + + helper::Log("Engine", "SscWriter", "SyncReadPattern", "", + m_Verbosity >= 10 ? m_WriterRank : 0, m_WriterRank, 5, + m_Verbosity, helper::LogMode::INFO); + + ssc::Buffer globalBuffer; + + ssc::BroadcastMetadata(globalBuffer, m_ReaderMasterStreamRank, + m_StreamComm); + + m_ReaderSelectionsLocked = globalBuffer[1]; + + ssc::Deserialize(globalBuffer, m_GlobalReadPattern, m_IO, false, false); + m_AllSendingReaderRanks = ssc::CalculateOverlap( + m_GlobalReadPattern, m_GlobalWritePattern[m_StreamRank]); + CalculatePosition(m_GlobalWritePattern, m_GlobalReadPattern, m_WriterRank, + m_AllSendingReaderRanks); + + if (m_Verbosity >= 10) + { + for (int i = 0; i < m_WriterSize; ++i) + { + MPI_Barrier(m_WriterComm); + if (i == m_WriterRank) + { + ssc::PrintRankPosMap(m_AllSendingReaderRanks, + "Rank Pos Map for Writer " + + std::to_string(m_WriterRank)); + } + } + MPI_Barrier(m_WriterComm); + } +} + +void SscWriterGeneric::CalculatePosition(ssc::BlockVecVec &writerVecVec, + ssc::BlockVecVec &readerVecVec, + const int writerRank, + ssc::RankPosMap &allOverlapRanks) +{ + for (auto &overlapRank : allOverlapRanks) + { + auto &readerRankMap = readerVecVec[overlapRank.first]; + auto currentReaderOverlapWriterRanks = + CalculateOverlap(writerVecVec, readerRankMap); + size_t bufferPosition = 0; + for (int rank = 0; rank < static_cast(writerVecVec.size()); ++rank) + { + bool hasOverlap = false; + for (const auto &r : currentReaderOverlapWriterRanks) + { + if (r.first == rank) + { + hasOverlap = true; + break; + } + } + if (hasOverlap) + { + currentReaderOverlapWriterRanks[rank].first = bufferPosition; + auto &bv = writerVecVec[rank]; + size_t currentRankTotalSize = TotalDataSize(bv) + 1; + currentReaderOverlapWriterRanks[rank].second = + currentRankTotalSize; + bufferPosition += currentRankTotalSize; + } + } + allOverlapRanks[overlapRank.first] = + currentReaderOverlapWriterRanks[writerRank]; + } +} + +} +} +} +} diff --git a/source/adios2/engine/ssc/SscWriterGeneric.h b/source/adios2/engine/ssc/SscWriterGeneric.h new file mode 100644 index 0000000000..24a63d58d7 --- /dev/null +++ b/source/adios2/engine/ssc/SscWriterGeneric.h @@ -0,0 +1,74 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * SscWriterGeneric.h + * + * Created on: Mar 3, 2022 + * Author: Jason Wang + */ + +#ifndef ADIOS2_ENGINE_SSCWRITERGENERIC_H_ +#define ADIOS2_ENGINE_SSCWRITERGENERIC_H_ + +#include "SscWriterBase.h" +#include "adios2/core/IO.h" +#include + +namespace adios2 +{ +namespace core +{ +namespace engine +{ +namespace ssc +{ + +class SscWriterGeneric : public SscWriterBase +{ + +public: + SscWriterGeneric(IO &io, const std::string &name, const Mode mode, + MPI_Comm comm); + ~SscWriterGeneric() = default; + + StepStatus BeginStep(const StepMode mode, const float timeoutSeconds, + const bool writerLocked) final; + size_t CurrentStep() final; + void PerformPuts() final; + void EndStep(const bool writerLocked) final; + void Close(const int transportIndex) final; + +#define declare_type(T) void PutDeferred(Variable &, const T *) final; + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + +private: + MPI_Win m_MpiWin; + std::thread m_EndStepThread; + ssc::BlockVecVec m_GlobalWritePattern; + ssc::BlockVecVec m_GlobalReadPattern; + std::vector m_MpiRequests; + ssc::RankPosMap m_AllSendingReaderRanks; + + bool m_WriterDefinitionsLocked = false; + bool m_ReaderSelectionsLocked = false; + + template + void PutDeferredCommon(Variable &variable, const T *values); + void SyncWritePattern(bool finalStep = false); + void SyncReadPattern(); + void EndStepFirst(); + void EndStepConsequentFixed(); + void EndStepConsequentFlexible(); + void CalculatePosition(ssc::BlockVecVec &writerMapVec, + ssc::BlockVecVec &readerMapVec, const int writerRank, + ssc::RankPosMap &allOverlapRanks); +}; + +} +} +} +} + +#endif // ADIOS2_ENGINE_SSCWRITERGENERIC_H_ diff --git a/source/adios2/engine/ssc/SscWriter.tcc b/source/adios2/engine/ssc/SscWriterGeneric.tcc similarity index 86% rename from source/adios2/engine/ssc/SscWriter.tcc rename to source/adios2/engine/ssc/SscWriterGeneric.tcc index e2c93aaedb..5f40a581ba 100644 --- a/source/adios2/engine/ssc/SscWriter.tcc +++ b/source/adios2/engine/ssc/SscWriterGeneric.tcc @@ -2,19 +2,16 @@ * Distributed under the OSI-approved Apache License, Version 2.0. See * accompanying file Copyright.txt for details. * - * SscWriter.tcc + * SscWriterGeneric.tcc * - * Created on: Nov 1, 2018 + * Created on: Mar 3, 2022 * Author: Jason Wang */ -#ifndef ADIOS2_ENGINE_SSCWRITER_TCC_ -#define ADIOS2_ENGINE_SSCWRITER_TCC_ +#ifndef ADIOS2_ENGINE_SSCWRITERGENERIC_TCC_ +#define ADIOS2_ENGINE_SSCWRITERGENERIC_TCC_ -#include "SscWriter.h" -#include "adios2/helper/adiosFunctions.h" -#include -#include +#include "SscWriterGeneric.h" namespace adios2 { @@ -22,12 +19,13 @@ namespace core { namespace engine { +namespace ssc +{ template <> -void SscWriter::PutDeferredCommon(Variable &variable, - const std::string *data) +void SscWriterGeneric::PutDeferredCommon(Variable &variable, + const std::string *data) { - PERFSTUBS_SCOPED_TIMER_FUNC(); variable.SetData(data); bool found = false; @@ -78,9 +76,8 @@ void SscWriter::PutDeferredCommon(Variable &variable, } template -void SscWriter::PutDeferredCommon(Variable &variable, const T *data) +void SscWriterGeneric::PutDeferredCommon(Variable &variable, const T *data) { - PERFSTUBS_SCOPED_TIMER_FUNC(); if ((variable.m_ShapeID == ShapeID::GlobalValue || variable.m_ShapeID == ShapeID::LocalValue || @@ -147,9 +144,9 @@ void SscWriter::PutDeferredCommon(Variable &variable, const T *data) } } } +} +} +} +} -} // end namespace engine -} // end namespace core -} // end namespace adios2 - -#endif // ADIOS2_ENGINE_SSCWRITER_TCC_ +#endif // ADIOS2_ENGINE_SSCWRITERGENERIC_TCC_ diff --git a/source/adios2/engine/ssc/SscWriterNaive.cpp b/source/adios2/engine/ssc/SscWriterNaive.cpp new file mode 100644 index 0000000000..604ea5f16f --- /dev/null +++ b/source/adios2/engine/ssc/SscWriterNaive.cpp @@ -0,0 +1,108 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * SscWriterNaive.cpp + * + * Created on: Mar 7, 2022 + * Author: Jason Wang + */ + +#include "SscWriterNaive.tcc" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ +namespace ssc +{ + +SscWriterNaive::SscWriterNaive(IO &io, const std::string &name, const Mode mode, + MPI_Comm comm) +: SscWriterBase(io, name, mode, comm) +{ +} + +StepStatus SscWriterNaive::BeginStep(const StepMode mode, + const float timeoutSeconds, + const bool writerLocked) +{ + ++m_CurrentStep; + + m_Buffer.clear(); + m_Buffer.resize(16); + m_Metadata.clear(); + + return StepStatus::OK; +} + +size_t SscWriterNaive::CurrentStep() { return m_CurrentStep; } + +void SscWriterNaive::PerformPuts() {} + +void SscWriterNaive::EndStep(const bool writerLocked) +{ + m_Buffer.value() = m_Buffer.size(); + m_Buffer.value(8) = m_Buffer.size(); + + ssc::SerializeVariables(m_Metadata, m_Buffer, m_WriterRank); + + if (m_WriterRank == 0) + { + ssc::SerializeAttributes(m_IO, m_Buffer); + } + + int localSize = static_cast(m_Buffer.value()); + std::vector localSizes(m_WriterSize); + MPI_Gather(&localSize, 1, MPI_INT, localSizes.data(), 1, MPI_INT, 0, + m_WriterComm); + size_t globalSize = + std::accumulate(localSizes.begin(), localSizes.end(), 0); + ssc::Buffer globalBuffer(globalSize); + + std::vector displs(m_WriterSize); + for (size_t i = 1; i < static_cast(m_WriterSize); ++i) + { + displs[i] = displs[i - 1] + localSizes[i - 1]; + } + + MPI_Gatherv(m_Buffer.data(), localSize, MPI_CHAR, globalBuffer.data(), + localSizes.data(), displs.data(), MPI_CHAR, 0, m_WriterComm); + + if (m_WriterRank == 0) + { + MPI_Send(&globalSize, 1, MPI_UNSIGNED_LONG_LONG, + m_ReaderMasterStreamRank, 0, m_StreamComm); + MPI_Send(globalBuffer.data(), globalSize, MPI_CHAR, + m_ReaderMasterStreamRank, 0, m_StreamComm); + } +} + +void SscWriterNaive::Close(const int transportIndex) +{ + + uint64_t globalSize = 1; + ssc::Buffer globalBuffer(globalSize); + if (m_WriterRank == 0) + { + MPI_Send(&globalSize, 1, MPI_UNSIGNED_LONG_LONG, + m_ReaderMasterStreamRank, 0, m_StreamComm); + MPI_Send(globalBuffer.data(), globalSize, MPI_CHAR, + m_ReaderMasterStreamRank, 0, m_StreamComm); + } +} + +#define declare_type(T) \ + void SscWriterNaive::PutDeferred(Variable &variable, const T *data) \ + { \ + PutDeferredCommon(variable, data); \ + } +ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + +} +} +} +} diff --git a/source/adios2/engine/ssc/SscWriterNaive.h b/source/adios2/engine/ssc/SscWriterNaive.h new file mode 100644 index 0000000000..0e6b9c6f71 --- /dev/null +++ b/source/adios2/engine/ssc/SscWriterNaive.h @@ -0,0 +1,58 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * SscWriterNaive.h + * + * Created on: Mar 7, 2022 + * Author: Jason Wang + */ + +#ifndef ADIOS2_ENGINE_SSCWRITERNAIVE_H_ +#define ADIOS2_ENGINE_SSCWRITERNAIVE_H_ + +#include "SscWriterBase.h" +#include "adios2/core/IO.h" +#include + +namespace adios2 +{ +namespace core +{ +namespace engine +{ +namespace ssc +{ + +class SscWriterNaive : public SscWriterBase +{ + +public: + SscWriterNaive(IO &io, const std::string &name, const Mode mode, + MPI_Comm comm); + ~SscWriterNaive() = default; + + StepStatus BeginStep(const StepMode mode, const float timeoutSeconds, + const bool writerLocked) final; + size_t CurrentStep() final; + void PerformPuts() final; + void EndStep(const bool writerLocked) final; + void Close(const int transportIndex) final; + +#define declare_type(T) void PutDeferred(Variable &, const T *) final; + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + +private: + template + void PutDeferredCommon(Variable &variable, const T *values); + + ssc::BlockVec m_Metadata; +}; + +} +} +} +} + +#endif // ADIOS2_ENGINE_SSCWRITENAIVE_H_ diff --git a/source/adios2/engine/ssc/SscWriterNaive.tcc b/source/adios2/engine/ssc/SscWriterNaive.tcc new file mode 100644 index 0000000000..c087a04a0d --- /dev/null +++ b/source/adios2/engine/ssc/SscWriterNaive.tcc @@ -0,0 +1,110 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * SscWriterNaive.tcc + * + * Created on: Mar 7, 2022 + * Author: Jason Wang + */ + +#ifndef ADIOS2_ENGINE_SSCWRITERNAIVE_TCC_ +#define ADIOS2_ENGINE_SSCWRITERNAIVE_TCC_ + +#include "SscWriterNaive.h" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ +namespace ssc +{ + +template <> +void SscWriterNaive::PutDeferredCommon(Variable &variable, + const std::string *data) +{ + variable.SetData(data); + m_Metadata.emplace_back(); + auto &b = m_Metadata.back(); + b.name = variable.m_Name; + b.type = DataType::String; + b.shapeId = variable.m_ShapeID; + b.shape = variable.m_Shape; + b.start = variable.m_Start; + b.count = variable.m_Count; + b.bufferStart = m_Buffer.size(); + b.bufferCount = data->size(); + m_Buffer.resize(b.bufferStart + b.bufferCount); + std::memcpy(m_Buffer.data() + b.bufferStart, data->data(), data->size()); + b.value.resize(data->size()); + std::memcpy(b.value.data(), data->data(), data->size()); +} + +template +void SscWriterNaive::PutDeferredCommon(Variable &variable, const T *data) +{ + + if ((variable.m_ShapeID == ShapeID::GlobalValue || + variable.m_ShapeID == ShapeID::LocalValue || + variable.m_Type == DataType::String) && + m_WriterRank != 0) + { + return; + } + + variable.SetData(data); + + Dims vStart = variable.m_Start; + Dims vCount = variable.m_Count; + Dims vShape = variable.m_Shape; + + if (m_IO.m_ArrayOrder != ArrayOrdering::RowMajor) + { + std::reverse(vStart.begin(), vStart.end()); + std::reverse(vCount.begin(), vCount.end()); + std::reverse(vShape.begin(), vShape.end()); + } + + bool found = false; + for (const auto &b : m_Metadata) + { + if (b.name == variable.m_Name && ssc::AreSameDims(vStart, b.start) && + ssc::AreSameDims(vCount, b.count) && + ssc::AreSameDims(vShape, b.shape)) + { + std::memcpy(m_Buffer.data() + b.bufferStart, data, b.bufferCount); + found = true; + } + } + + if (!found) + { + m_Metadata.emplace_back(); + auto &b = m_Metadata.back(); + b.name = variable.m_Name; + b.type = helper::GetDataType(); + b.shapeId = variable.m_ShapeID; + b.shape = vShape; + b.start = vStart; + b.count = vCount; + b.bufferStart = m_Buffer.size(); + b.bufferCount = ssc::TotalDataSize(b.count, b.type, b.shapeId); + m_Buffer.resize(b.bufferStart + b.bufferCount); + std::memcpy(m_Buffer.data() + b.bufferStart, data, b.bufferCount); + if (b.shapeId == ShapeID::GlobalValue || + b.shapeId == ShapeID::LocalValue) + { + b.value.resize(sizeof(*data)); + std::memcpy(b.value.data(), data, b.bufferCount); + } + } +} +} +} +} +} + +#endif // ADIOS2_ENGINE_SSCWRITERNAIVE_TCC_ diff --git a/source/adios2/helper/adiosMemory.cpp b/source/adios2/helper/adiosMemory.cpp index 7a8247ad40..98ff373072 100644 --- a/source/adios2/helper/adiosMemory.cpp +++ b/source/adios2/helper/adiosMemory.cpp @@ -451,8 +451,9 @@ int NdCopy(const char *in, const Dims &inStart, const Dims &inCount, #ifdef ADIOS2_HAVE_CUDA if (MemSpace == MemorySpace::CUDA) { - helper::CudaMemCopyFromBuffer(outOvlpBase, 0, inOvlpBase, - blockSize); + helper::NdCopyCUDA(inOvlpBase, outOvlpBase, inOvlpGapSize, + outOvlpGapSize, ovlpCount, minContDim, + blockSize); return 0; } #endif @@ -509,6 +510,14 @@ int NdCopy(const char *in, const Dims &inStart, const Dims &inCount, // padding else { +#ifdef ADIOS2_HAVE_CUDA + if (MemSpace == MemorySpace::CUDA) + { + helper::Throw( + "Helper", "Memory", "CopyContiguousMemory", + "Direct byte order reversal not supported for GPU buffers"); + } +#endif // Dims revInCount(inCount); // Dims revOutCount(outCount); // @@ -607,14 +616,6 @@ int NdCopy(const char *in, const Dims &inStart, const Dims &inCount, // Same Endian" if (inIsLittleEndian == outIsLittleEndian) { -#ifdef ADIOS2_HAVE_CUDA - if (MemSpace == MemorySpace::CUDA) - { - helper::CudaMemCopyFromBuffer(outOvlpBase, 0, inOvlpBase, - blockSize); - return 0; - } -#endif if (!safeMode) { NdCopyRecurDFNonSeqDynamic(0, inOvlpBase, outOvlpBase, @@ -632,14 +633,6 @@ int NdCopy(const char *in, const Dims &inStart, const Dims &inCount, // different Endian" else { -#ifdef ADIOS2_HAVE_CUDA - if (MemSpace == MemorySpace::CUDA) - { - helper::Throw( - "Helper", "Memory", "CopyContiguousMemory", - "Direct byte order reversal not supported for GPU buffers"); - } -#endif if (!safeMode) { NdCopyRecurDFNonSeqDynamicRevEndian( diff --git a/source/adios2/helper/adiosMemory.inl b/source/adios2/helper/adiosMemory.inl index 6f7a8ae3d3..c5a94187e9 100644 --- a/source/adios2/helper/adiosMemory.inl +++ b/source/adios2/helper/adiosMemory.inl @@ -98,6 +98,37 @@ void CudaMemCopyFromBuffer(T *GPUbuffer, size_t position, const char *source, char *dest = reinterpret_cast(GPUbuffer); MemcpyBufferToGPU(dest, source + position, size); } + +static inline void NdCopyCUDA(const char *&inOvlpBase, char *&outOvlpBase, + Dims &inOvlpGapSize, Dims &outOvlpGapSize, + Dims &ovlpCount, size_t minContDim, + size_t blockSize) +{ + Dims pos(ovlpCount.size(), 0); + size_t curDim = 0; + while (true) + { + while (curDim != minContDim) + { + pos[curDim]++; + curDim++; + } + CudaMemCopyFromBuffer(outOvlpBase, 0, inOvlpBase, blockSize); + inOvlpBase += blockSize; + outOvlpBase += blockSize; + do + { + if (curDim == 0) + { + return; + } + inOvlpBase += inOvlpGapSize[curDim]; + outOvlpBase += outOvlpGapSize[curDim]; + pos[curDim] = 0; + curDim--; + } while (pos[curDim] == ovlpCount[curDim]); + } +} #endif template diff --git a/source/adios2/operator/compress/CompressMGARDPlus.cpp b/source/adios2/operator/compress/CompressMGARDPlus.cpp index 13dc8728ac..24d70f04e3 100644 --- a/source/adios2/operator/compress/CompressMGARDPlus.cpp +++ b/source/adios2/operator/compress/CompressMGARDPlus.cpp @@ -10,6 +10,7 @@ #include "CompressMGARDPlus.h" #include "CompressMGARD.h" +#include "adios2/core/Engine.h" #include "adios2/helper/adiosFunctions.h" namespace adios2 @@ -28,6 +29,18 @@ size_t CompressMGARDPlus::Operate(const char *dataIn, const Dims &blockStart, const Dims &blockCount, const DataType type, char *bufferOut) { + + // Read ADIOS2 files from here + adios2::core::ADIOS adios("C++"); + auto &io = adios.DeclareIO("SubIO"); + auto *engine = &io.Open(m_Parameters["MeshFile"], adios2::Mode::Read); + auto var = io.InquireVariable(m_Parameters["MeshVariable"]); + std::vector data(std::accumulate(var->m_Shape.begin(), + var->m_Shape.end(), sizeof(float), + std::multiplies())); + engine->Get(*var, data); + // Read ADIOS2 files end, use data for your algorithm + size_t bufferOutOffset = 0; const uint8_t bufferVersion = 1; diff --git a/source/adios2/operator/compress/CompressSZ.cpp b/source/adios2/operator/compress/CompressSZ.cpp index c2fe37468b..f2363c6190 100644 --- a/source/adios2/operator/compress/CompressSZ.cpp +++ b/source/adios2/operator/compress/CompressSZ.cpp @@ -25,6 +25,8 @@ namespace core namespace compress { +std::mutex CompressSZ::m_Mutex; + CompressSZ::CompressSZ(const Params ¶meters) : Operator("sz", COMPRESS_SZ, "compress", parameters) { @@ -241,15 +243,6 @@ size_t CompressSZ::Operate(const char *dataIn, const Dims &blockStart, } } - if (use_configfile) - { - SZ_Init(sz_configfile.c_str()); - } - else - { - SZ_Init_Params(&sz); - } - // Get type info int dtype = -1; if (varType == helper::GetDataType() || @@ -272,9 +265,22 @@ size_t CompressSZ::Operate(const char *dataIn, const Dims &blockStart, convertedDims = ConvertDims(blockCount, varType, 5, true, 0); size_t szBufferSize; + + m_Mutex.lock(); + if (use_configfile) + { + SZ_Init(sz_configfile.c_str()); + } + else + { + SZ_Init_Params(&sz); + } auto *szBuffer = SZ_compress( dtype, const_cast(dataIn), &szBufferSize, convertedDims[0], convertedDims[1], convertedDims[2], convertedDims[3], convertedDims[4]); + SZ_Finalize(); + m_Mutex.unlock(); + if (bufferOutOffset + szBufferSize > helper::GetTotalSize(blockCount, helper::GetDataTypeSize(varType))) { @@ -291,7 +297,6 @@ size_t CompressSZ::Operate(const char *dataIn, const Dims &blockStart, { free(szBuffer); } - SZ_Finalize(); return bufferOutOffset; } @@ -391,12 +396,15 @@ size_t CompressSZ::DecompressV1(const char *bufferIn, const size_t sizeIn, const size_t dataSizeBytes = helper::GetTotalSize(convertedDims, dataTypeSize); + m_Mutex.lock(); void *result = SZ_decompress(dtype, reinterpret_cast( const_cast(bufferIn + bufferInOffset)), sizeIn - bufferInOffset, 0, convertedDims[0], convertedDims[1], convertedDims[2], convertedDims[3]); + SZ_Finalize(); + m_Mutex.unlock(); if (result == nullptr) { @@ -462,12 +470,15 @@ size_t CompressSZ::DecompressV2(const char *bufferIn, const size_t sizeIn, Dims convertedDims = ConvertDims(blockCount, helper::GetDataType(), 5, true, 0); + m_Mutex.lock(); void *result = SZ_decompress( dtype, reinterpret_cast( const_cast(bufferIn + bufferInOffset)), sizeIn - bufferInOffset, convertedDims[0], convertedDims[1], convertedDims[2], convertedDims[3], convertedDims[4]); + SZ_Finalize(); + m_Mutex.unlock(); if (result == nullptr) { diff --git a/source/adios2/operator/compress/CompressSZ.h b/source/adios2/operator/compress/CompressSZ.h index 789c0cebc5..20e08123fc 100644 --- a/source/adios2/operator/compress/CompressSZ.h +++ b/source/adios2/operator/compress/CompressSZ.h @@ -80,6 +80,7 @@ class CompressSZ : public Operator char *dataOut); std::string m_VersionInfo; + static std::mutex m_Mutex; }; } // end namespace compress diff --git a/testing/adios2/engine/bp/CMakeLists.txt b/testing/adios2/engine/bp/CMakeLists.txt index f3a76e61f1..029e138c74 100644 --- a/testing/adios2/engine/bp/CMakeLists.txt +++ b/testing/adios2/engine/bp/CMakeLists.txt @@ -164,14 +164,30 @@ gtest_add_tests_helper(StepsInSituLocalArray MPI_ALLOW BP Engine.BP. .FileStream WORKING_DIRECTORY ${FS_DIR} EXTRA_ARGS "FileStream" ) -if(ADIOS2_HAVE_CUDA AND ADIOS2_HAVE_BP5) +if(ADIOS2_HAVE_CUDA) enable_language(CUDA) - gtest_add_tests_helper(WriteReadCuda MPI_ALLOW BP Engine.BP. .BP5 - WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5" - ) + gtest_add_tests_helper(WriteReadCuda MPI_ALLOW BP Engine.BP. .BP4 + WORKING_DIRECTORY ${BP4_DIR} EXTRA_ARGS "BP4" + ) + gtest_add_tests_helper(SelectionsCuda MPI_ALLOW BP Engine.BP. .BP4 + WORKING_DIRECTORY ${BP4_DIR} EXTRA_ARGS "BP4" + ) + if(ADIOS2_HAVE_BP5) + gtest_add_tests_helper(WriteReadCuda MPI_ALLOW BP Engine.BP. .BP5 + WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5" + ) + gtest_add_tests_helper(SelectionsCuda MPI_ALLOW BP Engine.BP. .BP5 + WORKING_DIRECTORY ${BP5_DIR} EXTRA_ARGS "BP5" + ) + endif() foreach(tgt ${Test.Engine.BP.WriteReadCuda-TARGETS}) target_sources(${tgt} PRIVATE operations/CudaRoutines.cu) target_link_libraries(${tgt} CUDA::cudart) endforeach() + + foreach(tgt ${Test.Engine.BP.SelectionsCuda-TARGETS}) + target_sources(${tgt} PRIVATE operations/CudaRoutines.cu) + target_link_libraries(${tgt} CUDA::cudart) + endforeach() endif() diff --git a/testing/adios2/engine/bp/TestBPSelectionsCuda.cpp b/testing/adios2/engine/bp/TestBPSelectionsCuda.cpp new file mode 100644 index 0000000000..31c0c108a1 --- /dev/null +++ b/testing/adios2/engine/bp/TestBPSelectionsCuda.cpp @@ -0,0 +1,302 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ + +#include "operations/CudaRoutines.h" + +#include +#include + +#include +#include +#include +#include + +#include +#include + +#include + +std::string engineName; // comes from command line + +class ADIOSSelectionCUDATest : public ::testing::Test +{ +public: + ADIOSSelectionCUDATest() = default; +}; + +void copySelection2D(const double *a, const adios2::Dims &shape, + const adios2::Dims &start, const adios2::Dims &count, + double *b) +{ + double *bp = b; + for (size_t x = 0; x < count[0]; ++x) + { + for (size_t y = 0; y < count[1]; ++y) + { + const size_t aidx = (start[0] + x) * shape[1] + start[1] + y; + *bp = a[aidx]; + ++bp; + } + } +} + +bool compareSelection2D(const double *a, const adios2::Dims &shape, + const adios2::Dims &start, const adios2::Dims &count, + double *b, adios2::Dims &firstNonEqPoint) +{ + std::cout << " compare Block: shape = " << adios2::ToString(shape) + << " start = " << adios2::ToString(start) + << " count = " << adios2::ToString(count) << std::endl; + bool match = true; + double *bp = b; + for (size_t x = 0; x < count[0]; ++x) + { + size_t aidx = (start[0] + x) * shape[1] + start[1]; + for (size_t y = 0; y < count[1]; ++y) + { + if (*bp != a[aidx]) + { + firstNonEqPoint = {x, y}; + std::cout << " Non-match at pos = " + << adios2::ToString(firstNonEqPoint) + << " : a = " << a[aidx] << ", b = " << *bp + << std::endl; + match = false; + } + ++bp; + ++aidx; + } + } + return match; +} + +bool compareSelection2D_F(const double *a, const adios2::Dims &shape, + const adios2::Dims &start, const adios2::Dims &count, + double *b, adios2::Dims &firstNonEqPoint) +{ + std::cout << " compare Block: shape = " << adios2::ToString(shape) + << " start = " << adios2::ToString(start) + << " count = " << adios2::ToString(count) << std::endl; + bool match = true; + double *bp = b; + for (size_t y = 0; y < count[1]; ++y) + { + size_t aidx = (start[1] + y) * shape[0] + start[0]; + for (size_t x = 0; x < count[0]; ++x) + { + if (*bp != a[aidx]) + { + firstNonEqPoint = {y, x}; + std::cout << " Non-match at pos = " + << adios2::ToString(firstNonEqPoint) + << " : a = " << a[aidx] << ", b = " << *bp + << std::endl; + match = false; + } + ++bp; + ++aidx; + } + } + return match; +} + +TEST_F(ADIOSSelectionCUDATest, 2D) +{ + constexpr size_t C1 = 5; + constexpr size_t C2 = 4; + constexpr size_t DIM1 = 3 * C1; + constexpr size_t DIM2 = 3 * C2; + const std::string filename = "ADIOSSelectionCuda2D.bp"; + double a[DIM1 * DIM2]; + + double *ap = a; + for (size_t x = 1; x <= DIM1; ++x) + { + for (size_t y = 1; y <= DIM2; ++y) + { + *ap = x * 1.0 + y / 100.0; + ++ap; + } + } + /* ./bin/bpls -la testing/adios2/engine/bp/bp4/ADIOSSelection2D.bp + -d a -n 12 -f "%6.3f" + */ + + // Write test data using BP + { + adios2::ADIOS adios; + adios2::IO ioWrite = adios.DeclareIO("TestIOWrite"); + if (!engineName.empty()) + { + ioWrite.SetEngine(engineName); + } + else + { + // Create the BP Engine + ioWrite.SetEngine("BPFile"); + } + + std::cout << "Write data as CUDA buffer..." << std::endl; + + adios2::Engine engine = ioWrite.Open(filename, adios2::Mode::Write); + const adios2::Dims shape = {DIM1, DIM2}; + const adios2::Dims count = {C1, C2}; + adios2::Dims start{0, 0}; + + double b[C1 * C2]; + + adios2::Variable var = + ioWrite.DefineVariable("a", shape, start, count); + + /*adios2::Variable vara1 = + ioWrite.DefineVariable("a1", shape, start, shape);*/ + + engine.BeginStep(); + + double *gpuSimData; + cudaMalloc(&gpuSimData, C1 * C2 * sizeof(double)); + for (size_t x = 0; x < DIM1; x += count[0]) + { + for (size_t y = 0; y < DIM2; y += count[1]) + { + start = {x, y}; + copySelection2D(a, shape, start, count, b); + var.SetSelection({start, count}); + cudaMemcpy(gpuSimData, b, C1 * C2 * sizeof(double), + cudaMemcpyHostToDevice); + var.SetMemorySpace(adios2::MemorySpace::CUDA); + engine.Put(var, gpuSimData, adios2::Mode::Sync); + } + } + + engine.EndStep(); + engine.Close(); + } + + // Read data + { + adios2::ADIOS adios; + adios2::IO ioRead = adios.DeclareIO("TestIORead"); + ioRead.SetEngine("File"); + adios2::Engine engine = ioRead.Open(filename, adios2::Mode::Read); + EXPECT_TRUE(engine); + engine.BeginStep(); + adios2::Variable var = ioRead.InquireVariable("a"); + EXPECT_TRUE(var); + const adios2::Dims shape = {DIM1, DIM2}; + const adios2::Dims count = {C1, C2}; + adios2::Dims s{0, 0}; + adios2::Dims c = shape; + adios2::Dims firstNonMatch{0, 0}; + + std::cout << "CUDA read selections with entire blocks..." << std::endl; + + /* Entire array */ + { + std::vector res(DIM1 * DIM2); + double *gpuRet; + cudaMalloc(&gpuRet, DIM1 * DIM2 * sizeof(double)); + var.SetSelection({s, c}); + var.SetMemorySpace(adios2::MemorySpace::CUDA); + engine.Get(var, gpuRet, adios2::Mode::Sync); + cudaMemcpy(res.data(), gpuRet, DIM1 * DIM2 * sizeof(double), + cudaMemcpyDeviceToHost); + EXPECT_EQ(res.size(), DIM1 * DIM2); + EXPECT_TRUE( + compareSelection2D(a, shape, s, c, res.data(), firstNonMatch)); + } + + /* Single block in the center */ + { + s = {5, 4}; + c = count; + std::vector res(c[0] * c[1]); + double *gpuRet; + cudaMalloc(&gpuRet, c[0] * c[1] * sizeof(double)); + var.SetSelection({s, c}); + var.SetMemorySpace(adios2::MemorySpace::CUDA); + engine.Get(var, gpuRet, adios2::Mode::Sync); + cudaMemcpy(res.data(), gpuRet, c[0] * c[1] * sizeof(double), + cudaMemcpyDeviceToHost); + EXPECT_EQ(res.size(), c[0] * c[1]); + EXPECT_TRUE( + compareSelection2D(a, shape, s, c, res.data(), firstNonMatch)); + } + + /* Four blocks in X-Y direction */ + { + s = {5, 4}; + c = {2 * count[0], 2 * count[1]}; + std::vector res(c[0] * c[1]); + double *gpuRet; + cudaMalloc(&gpuRet, c[0] * c[1] * sizeof(double)); + var.SetSelection({s, c}); + var.SetMemorySpace(adios2::MemorySpace::CUDA); + engine.Get(var, gpuRet, adios2::Mode::Sync); + cudaMemcpy(res.data(), gpuRet, c[0] * c[1] * sizeof(double), + cudaMemcpyDeviceToHost); + EXPECT_EQ(res.size(), c[0] * c[1]); + EXPECT_TRUE( + compareSelection2D(a, shape, s, c, res.data(), firstNonMatch)); + } + + /* + * Partial blocks + */ + + std::cout << "CUDA read selections with partial blocks..." << std::endl; + + /* center part of single block in center */ + { + s = {6, 5}; + c = {count[0] - 2, count[1] - 2}; + std::vector res(c[0] * c[1]); + double *gpuRet; + cudaMalloc(&gpuRet, c[0] * c[1] * sizeof(double)); + var.SetSelection({s, c}); + var.SetMemorySpace(adios2::MemorySpace::CUDA); + engine.Get(var, gpuRet, adios2::Mode::Sync); + cudaMemcpy(res.data(), gpuRet, c[0] * c[1] * sizeof(double), + cudaMemcpyDeviceToHost); + EXPECT_EQ(res.size(), c[0] * c[1]); + EXPECT_TRUE( + compareSelection2D(a, shape, s, c, res.data(), firstNonMatch)); + } + + /* Center block plus 1 in each direction, cutting into all blocks */ + /* ./bin/bpls -la testing/adios2/engine/bp/bp4/ADIOSSelection2D.bp + -d a -f "%6.3f " -s "4,3" -c "7,6" -n 6 + */ + { + s = {4, 3}; + c = {count[0] + 2, count[1] + 2}; + std::vector res(c[0] * c[1]); + double *gpuRet; + cudaMalloc(&gpuRet, c[0] * c[1] * sizeof(double)); + var.SetSelection({s, c}); + var.SetMemorySpace(adios2::MemorySpace::CUDA); + engine.Get(var, gpuRet, adios2::Mode::Sync); + cudaMemcpy(res.data(), gpuRet, c[0] * c[1] * sizeof(double), + cudaMemcpyDeviceToHost); + EXPECT_EQ(res.size(), c[0] * c[1]); + EXPECT_TRUE( + compareSelection2D(a, shape, s, c, res.data(), firstNonMatch)); + } + + engine.EndStep(); + engine.Close(); + } +} + +int main(int argc, char **argv) +{ + ::testing::InitGoogleTest(&argc, argv); + if (argc > 1) + { + engineName = std::string(argv[1]); + } + int result = RUN_ALL_TESTS(); + return result; +} diff --git a/testing/adios2/engine/dataman/TestDataMan2DSz.cpp b/testing/adios2/engine/dataman/TestDataMan2DSz.cpp index 37fb19543a..4b6ec25270 100644 --- a/testing/adios2/engine/dataman/TestDataMan2DSz.cpp +++ b/testing/adios2/engine/dataman/TestDataMan2DSz.cpp @@ -311,7 +311,7 @@ TEST_F(DataManEngineTest, 2D_Sz) Dims start = {0, 0}; Dims count = {10, 10}; - size_t steps = 100; + size_t steps = 5000; adios2::Params engineParams = { {"IPAddress", "127.0.0.1"}, {"Port", "12330"}, {"Verbose", "0"}}; diff --git a/testing/adios2/engine/ssc/CMakeLists.txt b/testing/adios2/engine/ssc/CMakeLists.txt index 1289f536e8..ce251c4e89 100644 --- a/testing/adios2/engine/ssc/CMakeLists.txt +++ b/testing/adios2/engine/ssc/CMakeLists.txt @@ -68,4 +68,7 @@ endif() gtest_add_tests_helper(VaryingSteps MPI_ONLY Ssc Engine.SSC. "") SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscVaryingSteps.MPI "" TRUE) + gtest_add_tests_helper(ZeroBlock MPI_ONLY Ssc Engine.SSC. "") + SetupTestPipeline(Engine.SSC.SscEngineTest.TestSscZeroBlock.MPI "" TRUE) + endif() diff --git a/testing/adios2/engine/ssc/TestSsc7d.cpp b/testing/adios2/engine/ssc/TestSsc7d.cpp index 730979421e..6238e7f1f7 100644 --- a/testing/adios2/engine/ssc/TestSsc7d.cpp +++ b/testing/adios2/engine/ssc/TestSsc7d.cpp @@ -192,36 +192,72 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSsc7d) { - std::string filename = "TestSsc7d"; - adios2::Params engineParams = {}; + { + std::string filename = "TestSsc7d"; + adios2::Params engineParams = {}; - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - int mpiGroup = worldRank / (worldSize / 2); - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); - Dims shape = {10, 2, 2, (size_t)mpiSize, 2, 8, 10}; - Dims start = {0, 0, 0, (size_t)mpiRank, 0, 0, 0}; - Dims count = {10, 2, 2, 1, 2, 8, 10}; - size_t steps = 20; + Dims shape = {10, 2, 2, (size_t)mpiSize, 2, 8, 10}; + Dims start = {0, 0, 0, (size_t)mpiRank, 0, 0, 0}; + Dims count = {10, 2, 2, 1, 2, 8, 10}; + size_t steps = 20; - if (mpiGroup == 0) - { - Writer(shape, start, count, steps, engineParams, filename); - } + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); - if (mpiGroup == 1) - { - Reader(shape, start, count, steps, engineParams, filename); + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); } - MPI_Barrier(MPI_COMM_WORLD); + { + std::string filename = "TestSsc7dNaive"; + adios2::Params engineParams = {{"Verbose", "0"}, + {"EngineMode", "naive"}}; + + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + Dims shape = {10, 2, 2, (size_t)mpiSize, 2, 8, 10}; + Dims start = {0, 0, 0, (size_t)mpiRank, 0, 0, 0}; + Dims count = {10, 2, 2, 1, 2, 8, 10}; + size_t steps = 20; + + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); + } } int main(int argc, char **argv) diff --git a/testing/adios2/engine/ssc/TestSscBase.cpp b/testing/adios2/engine/ssc/TestSscBase.cpp index 0409ff9a61..6ce18c2cc5 100644 --- a/testing/adios2/engine/ssc/TestSscBase.cpp +++ b/testing/adios2/engine/ssc/TestSscBase.cpp @@ -242,36 +242,57 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSscBase) { - std::string filename = "TestSscBase"; - adios2::Params engineParams = {{"Verbose", "0"}}; - - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - int mpiGroup = worldRank / (worldSize / 2); - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); - - Dims shape = {10, (size_t)mpiSize * 2}; - Dims start = {2, (size_t)mpiRank * 2}; - Dims count = {5, 2}; - size_t steps = 100; - - if (mpiGroup == 0) { - Writer(shape, start, count, steps, engineParams, filename); + std::string filename = "TestSscBase"; + adios2::Params engineParams = {{"Verbose", "0"}}; + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + Dims shape = {10, (size_t)mpiSize * 2}; + Dims start = {2, (size_t)mpiRank * 2}; + Dims count = {5, 2}; + size_t steps = 100; + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + MPI_Barrier(MPI_COMM_WORLD); } - - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - - if (mpiGroup == 1) { - Reader(shape, start, count, steps, engineParams, filename); + std::string filename = "TestSscBaseNaive"; + adios2::Params engineParams = {{"Verbose", "0"}, + {"EngineMode", "naive"}}; + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + Dims shape = {10, (size_t)mpiSize * 2}; + Dims start = {2, (size_t)mpiRank * 2}; + Dims count = {5, 2}; + size_t steps = 100; + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + MPI_Barrier(MPI_COMM_WORLD); } - - MPI_Barrier(MPI_COMM_WORLD); } int main(int argc, char **argv) diff --git a/testing/adios2/engine/ssc/TestSscMoreReadersThanWriters.cpp b/testing/adios2/engine/ssc/TestSscMoreReadersThanWriters.cpp index 5f0619e80e..0064818722 100644 --- a/testing/adios2/engine/ssc/TestSscMoreReadersThanWriters.cpp +++ b/testing/adios2/engine/ssc/TestSscMoreReadersThanWriters.cpp @@ -193,52 +193,104 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSscMoreReadersThanWriters) { - std::string filename = "TestSscMoreReadersThanWriters"; - adios2::Params engineParams = {}; - - int worldRank, worldSize; - Dims start, count, shape; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - int mpiGroup; - int writers = 2; - if (worldSize < 3) - { - return; - } - if (worldRank < writers) { - mpiGroup = 0; - } - else - { - mpiGroup = 1; - } + std::string filename = "TestSscMoreReadersThanWriters"; + adios2::Params engineParams = {}; - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + int worldRank, worldSize; + Dims start, count, shape; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup; + int writers = 2; + if (worldSize < 3) + { + return; + } + if (worldRank < writers) + { + mpiGroup = 0; + } + else + { + mpiGroup = 1; + } - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - size_t steps = 20; + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); - if (mpiGroup == 0) - { - shape = {(size_t)writers, 10}; - start = {(size_t)mpiRank, 0}; - count = {1, 10}; - Writer(shape, start, count, steps, engineParams, filename); + size_t steps = 20; + + if (mpiGroup == 0) + { + shape = {(size_t)writers, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + Writer(shape, start, count, steps, engineParams, filename); + } + + if (mpiGroup == 1) + { + shape = {(size_t)writers, 10}; + start = {0, 0}; + count = shape; + Reader(shape, start, shape, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); } - if (mpiGroup == 1) { - shape = {(size_t)writers, 10}; - start = {0, 0}; - count = shape; - Reader(shape, start, shape, steps, engineParams, filename); - } + std::string filename = "TestSscMoreReadersThanWritersNaive"; + adios2::Params engineParams = {{"Verbose", "0"}, + {"EngineMode", "naive"}}; + + int worldRank, worldSize; + Dims start, count, shape; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup; + int writers = 2; + if (worldSize < 3) + { + return; + } + if (worldRank < writers) + { + mpiGroup = 0; + } + else + { + mpiGroup = 1; + } - MPI_Barrier(MPI_COMM_WORLD); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + size_t steps = 20; + + if (mpiGroup == 0) + { + shape = {(size_t)writers, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + Writer(shape, start, count, steps, engineParams, filename); + } + + if (mpiGroup == 1) + { + shape = {(size_t)writers, 10}; + start = {0, 0}; + count = shape; + Reader(shape, start, shape, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); + } } int main(int argc, char **argv) diff --git a/testing/adios2/engine/ssc/TestSscSingleStep.cpp b/testing/adios2/engine/ssc/TestSscSingleStep.cpp index bf8b673788..6c4fe042a2 100644 --- a/testing/adios2/engine/ssc/TestSscSingleStep.cpp +++ b/testing/adios2/engine/ssc/TestSscSingleStep.cpp @@ -243,36 +243,72 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSscSingleStep) { - std::string filename = "TestSscSingleStep"; - adios2::Params engineParams = {{"Verbose", "0"}}; + { + std::string filename = "TestSscSingleStep"; + adios2::Params engineParams = {{"Verbose", "0"}}; - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - int mpiGroup = worldRank / (worldSize / 2); - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); - Dims shape = {10, (size_t)mpiSize * 2}; - Dims start = {2, (size_t)mpiRank * 2}; - Dims count = {5, 2}; - size_t steps = 1; + Dims shape = {10, (size_t)mpiSize * 2}; + Dims start = {2, (size_t)mpiRank * 2}; + Dims count = {5, 2}; + size_t steps = 1; - if (mpiGroup == 0) - { - Writer(shape, start, count, steps, engineParams, filename); - } + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); - if (mpiGroup == 1) - { - Reader(shape, start, count, steps, engineParams, filename); + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); } - MPI_Barrier(MPI_COMM_WORLD); + { + std::string filename = "TestSscSingleStepNaive"; + adios2::Params engineParams = {{"Verbose", "0"}, + {"EngineMode", "naive"}}; + + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + Dims shape = {10, (size_t)mpiSize * 2}; + Dims start = {2, (size_t)mpiRank * 2}; + Dims count = {5, 2}; + size_t steps = 1; + + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); + } } int main(int argc, char **argv) diff --git a/testing/adios2/engine/ssc/TestSscSuperLarge.cpp b/testing/adios2/engine/ssc/TestSscSuperLarge.cpp index f485658c42..4bd0b60606 100644 --- a/testing/adios2/engine/ssc/TestSscSuperLarge.cpp +++ b/testing/adios2/engine/ssc/TestSscSuperLarge.cpp @@ -242,36 +242,72 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSscSuperLarge) { - std::string filename = "TestSscSuperLarge"; - adios2::Params engineParams = {{"Verbose", "0"}}; + { + std::string filename = "TestSscSuperLarge"; + adios2::Params engineParams = {{"Verbose", "0"}}; - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - int mpiGroup = worldRank / (worldSize / 2); - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); - Dims shape = {100, (size_t)mpiSize * 10}; - Dims start = {10, (size_t)mpiRank * 10}; - Dims count = {90, 10}; - size_t steps = 50; + Dims shape = {100, (size_t)mpiSize * 10}; + Dims start = {10, (size_t)mpiRank * 10}; + Dims count = {90, 10}; + size_t steps = 50; - if (mpiGroup == 0) - { - Writer(shape, start, count, steps, engineParams, filename); - } + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); - if (mpiGroup == 1) - { - Reader(shape, start, count, steps, engineParams, filename); + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); } - MPI_Barrier(MPI_COMM_WORLD); + { + std::string filename = "TestSscSuperLargeNaive"; + adios2::Params engineParams = {{"Verbose", "0"}, + {"EngineMode", "naive"}}; + + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + Dims shape = {100, (size_t)mpiSize * 10}; + Dims start = {10, (size_t)mpiRank * 10}; + Dims count = {90, 10}; + size_t steps = 50; + + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); + } } int main(int argc, char **argv) diff --git a/testing/adios2/engine/ssc/TestSscUnbalanced.cpp b/testing/adios2/engine/ssc/TestSscUnbalanced.cpp index 17be4181ac..39aab4b19b 100644 --- a/testing/adios2/engine/ssc/TestSscUnbalanced.cpp +++ b/testing/adios2/engine/ssc/TestSscUnbalanced.cpp @@ -215,38 +215,76 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSscUnbalanced) { - std::string filename = "TestSscUnbalanced"; - adios2::Params engineParams = {{"Verbose", "0"}}; + { + std::string filename = "TestSscUnbalanced"; + adios2::Params engineParams = {{"Verbose", "0"}}; - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - int mpiGroup = worldRank / (worldSize / 2); - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); - Dims shape = {10, (size_t)mpiSize * 2}; - Dims start = {2, (size_t)mpiRank * 2}; - Dims count = {5, 2}; - size_t steps = 100; + Dims shape = {10, (size_t)mpiSize * 2}; + Dims start = {2, (size_t)mpiRank * 2}; + Dims count = {5, 2}; + size_t steps = 100; - if (mpiGroup == 0) - { - Writer(shape, start, count, steps, engineParams, filename); - } + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); - if (mpiGroup == 1) - { - start = {2, 0}; - count = {5, 2}; - Reader(shape, start, count, steps, engineParams, filename); + if (mpiGroup == 1) + { + start = {2, 0}; + count = {5, 2}; + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); } - MPI_Barrier(MPI_COMM_WORLD); + { + std::string filename = "TestSscUnbalancedNaive"; + adios2::Params engineParams = {{"Verbose", "0"}, + {"EngineMode", "naive"}}; + + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + Dims shape = {10, (size_t)mpiSize * 2}; + Dims start = {2, (size_t)mpiRank * 2}; + Dims count = {5, 2}; + size_t steps = 100; + + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + if (mpiGroup == 1) + { + start = {2, 0}; + count = {5, 2}; + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); + } } int main(int argc, char **argv) diff --git a/testing/adios2/engine/ssc/TestSscVaryingSteps.cpp b/testing/adios2/engine/ssc/TestSscVaryingSteps.cpp index f80b1c121f..639ca67522 100644 --- a/testing/adios2/engine/ssc/TestSscVaryingSteps.cpp +++ b/testing/adios2/engine/ssc/TestSscVaryingSteps.cpp @@ -288,36 +288,72 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSscVaryingSteps) { - std::string filename = "TestSscVaryingSteps"; - adios2::Params engineParams = {}; + { + std::string filename = "TestSscVaryingSteps"; + adios2::Params engineParams = {}; - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - int mpiGroup = worldRank / (worldSize / 2); - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); - Dims shape = {1, (size_t)mpiSize * 2}; - Dims start = {0, (size_t)mpiRank * 2}; - Dims count = {1, 2}; - size_t steps = 100; + Dims shape = {1, (size_t)mpiSize * 2}; + Dims start = {0, (size_t)mpiRank * 2}; + Dims count = {1, 2}; + size_t steps = 100; - if (mpiGroup == 0) - { - Writer(shape, start, count, steps, engineParams, filename); - } + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); - if (mpiGroup == 1) - { - Reader(shape, start, count, steps, engineParams, filename); + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); } - MPI_Barrier(MPI_COMM_WORLD); + { + std::string filename = "TestSscVaryingStepsNaive"; + adios2::Params engineParams = {{"Verbose", "0"}, + {"EngineMode", "naive"}}; + + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + Dims shape = {1, (size_t)mpiSize * 2}; + Dims start = {0, (size_t)mpiRank * 2}; + Dims count = {1, 2}; + size_t steps = 100; + + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); + } } int main(int argc, char **argv) diff --git a/testing/adios2/engine/ssc/TestSscWriterMultiblock.cpp b/testing/adios2/engine/ssc/TestSscWriterMultiblock.cpp index 02fa6cdfe6..b0d376d19f 100644 --- a/testing/adios2/engine/ssc/TestSscWriterMultiblock.cpp +++ b/testing/adios2/engine/ssc/TestSscWriterMultiblock.cpp @@ -255,39 +255,78 @@ void Reader(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSscWriterMultiblock) { - std::string filename = "TestSscWriterMultiblock"; - adios2::Params engineParams = {{"Verbose", "0"}}; + { + std::string filename = "TestSscWriterMultiblock"; + adios2::Params engineParams = {{"Verbose", "0"}}; - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - int mpiGroup = worldRank / (worldSize / 2); - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); - size_t steps = 100; + size_t steps = 100; - if (mpiGroup == 0) - { - Dims shape = {(size_t)mpiSize * 2, 10}; - Dims start = {(size_t)mpiRank * 2, 0}; - Dims count = {1, 10}; - Writer(shape, start, count, steps, engineParams, filename); - } + if (mpiGroup == 0) + { + Dims shape = {(size_t)mpiSize * 2, 10}; + Dims start = {(size_t)mpiRank * 2, 0}; + Dims count = {1, 10}; + Writer(shape, start, count, steps, engineParams, filename); + } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); - if (mpiGroup == 1) - { - Dims shape = {(size_t)mpiSize * 2, 10}; - Dims start = {0, 0}; - Dims count = {(size_t)mpiSize * 2, 10}; - Reader(shape, start, count, steps, engineParams, filename); + if (mpiGroup == 1) + { + Dims shape = {(size_t)mpiSize * 2, 10}; + Dims start = {0, 0}; + Dims count = {(size_t)mpiSize * 2, 10}; + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); } - MPI_Barrier(MPI_COMM_WORLD); + { + std::string filename = "TestSscWriterMultiblockNaive"; + adios2::Params engineParams = {{"Verbose", "0"}, + {"EngineMode", "naive"}}; + + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + size_t steps = 100; + + if (mpiGroup == 0) + { + Dims shape = {(size_t)mpiSize * 2, 10}; + Dims start = {(size_t)mpiRank * 2, 0}; + Dims count = {1, 10}; + Writer(shape, start, count, steps, engineParams, filename); + } + + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + + if (mpiGroup == 1) + { + Dims shape = {(size_t)mpiSize * 2, 10}; + Dims start = {0, 0}; + Dims count = {(size_t)mpiSize * 2, 10}; + Reader(shape, start, count, steps, engineParams, filename); + } + + MPI_Barrier(MPI_COMM_WORLD); + } } int main(int argc, char **argv) diff --git a/testing/adios2/engine/ssc/TestSscXgc2Way.cpp b/testing/adios2/engine/ssc/TestSscXgc2Way.cpp index 8774526b90..23005a39be 100644 --- a/testing/adios2/engine/ssc/TestSscXgc2Way.cpp +++ b/testing/adios2/engine/ssc/TestSscXgc2Way.cpp @@ -130,57 +130,115 @@ void gene(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSscXgc2Way) { - Dims start, count, shape; - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - - if (worldSize < 6) { - return; + Dims start, count, shape; + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + + if (worldSize < 6) + { + return; + } + + if (worldRank < 2) + { + mpiGroup = 0; + } + else + { + mpiGroup = 1; + } + + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + size_t steps = 20; + + if (mpiGroup == 0) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "2"}, + {"OpenTimeoutSecs", "3"}, + {"Verbose", "0"}}; + xgc(shape, start, count, steps, engineParams); + } + + if (mpiGroup == 1) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "2"}, + {"OpenTimeoutSecs", "3"}, + {"Verbose", "0"}}; + gene(shape, start, shape, steps, engineParams); + } + + MPI_Barrier(MPI_COMM_WORLD); } - if (worldRank < 2) - { - mpiGroup = 0; - } - else { - mpiGroup = 1; + Dims start, count, shape; + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + + if (worldSize < 6) + { + return; + } + + if (worldRank < 2) + { + mpiGroup = 0; + } + else + { + mpiGroup = 1; + } + + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + size_t steps = 20; + + if (mpiGroup == 0) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "2"}, + {"OpenTimeoutSecs", "3"}, + {"EngineMode", "naive"}, + {"Verbose", "0"}}; + xgc(shape, start, count, steps, engineParams); + } + + if (mpiGroup == 1) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "2"}, + {"OpenTimeoutSecs", "3"}, + {"EngineMode", "naive"}, + {"Verbose", "0"}}; + gene(shape, start, shape, steps, engineParams); + } + + MPI_Barrier(MPI_COMM_WORLD); } - - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); - - size_t steps = 20; - - if (mpiGroup == 0) - { - shape = {(size_t)mpiSize, 10}; - start = {(size_t)mpiRank, 0}; - count = {1, 10}; - adios2::Params engineParams = {{"RendezvousAppCount", "2"}, - {"MaxStreamsPerApp", "2"}, - {"OpenTimeoutSecs", "3"}, - {"Verbose", "0"}}; - xgc(shape, start, count, steps, engineParams); - } - - if (mpiGroup == 1) - { - shape = {(size_t)mpiSize, 10}; - start = {(size_t)mpiRank, 0}; - count = {1, 10}; - adios2::Params engineParams = {{"RendezvousAppCount", "2"}, - {"MaxStreamsPerApp", "2"}, - {"OpenTimeoutSecs", "3"}, - {"Verbose", "0"}}; - gene(shape, start, shape, steps, engineParams); - } - - MPI_Barrier(MPI_COMM_WORLD); } int main(int argc, char **argv) diff --git a/testing/adios2/engine/ssc/TestSscXgc3Way.cpp b/testing/adios2/engine/ssc/TestSscXgc3Way.cpp index ecbccb576f..daa2c4510b 100644 --- a/testing/adios2/engine/ssc/TestSscXgc3Way.cpp +++ b/testing/adios2/engine/ssc/TestSscXgc3Way.cpp @@ -223,73 +223,146 @@ void gene(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSscXgc3Way) { - Dims start, count, shape; - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - - if (worldSize < 4) - { - return; - } - - if (worldRank == 0) - { - mpiGroup = 0; - } - else if (worldRank == 1) - { - mpiGroup = 1; - } - else - { - mpiGroup = 2; - } - - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); - - size_t steps = 20; - - if (mpiGroup == 0) { - shape = {(size_t)mpiSize, 10}; - start = {(size_t)mpiRank, 0}; - count = {1, 10}; - adios2::Params engineParams = {{"RendezvousAppCount", "2"}, - {"MaxStreamsPerApp", "4"}, - {"OpenTimeoutSecs", "3"}, - {"Verbose", "0"}}; - coupler(shape, start, count, steps, engineParams); + Dims start, count, shape; + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + + if (worldSize < 4) + { + return; + } + + if (worldRank == 0) + { + mpiGroup = 0; + } + else if (worldRank == 1) + { + mpiGroup = 1; + } + else + { + mpiGroup = 2; + } + + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + size_t steps = 20; + + if (mpiGroup == 0) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"Verbose", "0"}}; + coupler(shape, start, count, steps, engineParams); + } + + if (mpiGroup == 1) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"Verbose", "0"}}; + gene(shape, start, shape, steps, engineParams); + } + + if (mpiGroup == 2) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"Verbose", "0"}}; + xgc(shape, start, count, steps, engineParams); + } + MPI_Barrier(MPI_COMM_WORLD); } - if (mpiGroup == 1) { - shape = {(size_t)mpiSize, 10}; - start = {(size_t)mpiRank, 0}; - count = {1, 10}; - adios2::Params engineParams = {{"RendezvousAppCount", "2"}, - {"MaxStreamsPerApp", "4"}, - {"OpenTimeoutSecs", "3"}, - {"Verbose", "0"}}; - gene(shape, start, shape, steps, engineParams); + Dims start, count, shape; + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + + if (worldSize < 4) + { + return; + } + + if (worldRank == 0) + { + mpiGroup = 0; + } + else if (worldRank == 1) + { + mpiGroup = 1; + } + else + { + mpiGroup = 2; + } + + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + size_t steps = 20; + + if (mpiGroup == 0) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"EngineMode", "naive"}, + {"Verbose", "0"}}; + coupler(shape, start, count, steps, engineParams); + } + + if (mpiGroup == 1) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"EngineMode", "naive"}, + {"Verbose", "0"}}; + gene(shape, start, shape, steps, engineParams); + } + + if (mpiGroup == 2) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"EngineMode", "naive"}, + {"Verbose", "0"}}; + xgc(shape, start, count, steps, engineParams); + } + MPI_Barrier(MPI_COMM_WORLD); } - - if (mpiGroup == 2) - { - shape = {(size_t)mpiSize, 10}; - start = {(size_t)mpiRank, 0}; - count = {1, 10}; - adios2::Params engineParams = {{"RendezvousAppCount", "2"}, - {"MaxStreamsPerApp", "4"}, - {"OpenTimeoutSecs", "3"}, - {"Verbose", "0"}}; - xgc(shape, start, count, steps, engineParams); - } - - MPI_Barrier(MPI_COMM_WORLD); } int main(int argc, char **argv) diff --git a/testing/adios2/engine/ssc/TestSscXgc3WayMatchedSteps.cpp b/testing/adios2/engine/ssc/TestSscXgc3WayMatchedSteps.cpp index df381ea4ab..ff050051a2 100644 --- a/testing/adios2/engine/ssc/TestSscXgc3WayMatchedSteps.cpp +++ b/testing/adios2/engine/ssc/TestSscXgc3WayMatchedSteps.cpp @@ -219,73 +219,148 @@ void gene(const Dims &shape, const Dims &start, const Dims &count, TEST_F(SscEngineTest, TestSscXgc3WayMatchedSteps) { - Dims start, count, shape; - int worldRank, worldSize; - MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); - MPI_Comm_size(MPI_COMM_WORLD, &worldSize); - - if (worldSize < 4) - { - return; - } - - if (worldRank == 0) - { - mpiGroup = 0; - } - else if (worldRank == 1) - { - mpiGroup = 1; - } - else - { - mpiGroup = 2; - } - - MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); - - MPI_Comm_rank(mpiComm, &mpiRank); - MPI_Comm_size(mpiComm, &mpiSize); - - size_t steps = 20; - - if (mpiGroup == 0) { - shape = {(size_t)mpiSize, 10}; - start = {(size_t)mpiRank, 0}; - count = {1, 10}; - adios2::Params engineParams = {{"RendezvousAppCount", "2"}, - {"MaxStreamsPerApp", "4"}, - {"OpenTimeoutSecs", "3"}, - {"Verbose", "0"}}; - coupler(shape, start, count, steps, engineParams); + Dims start, count, shape; + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + + if (worldSize < 4) + { + return; + } + + if (worldRank == 0) + { + mpiGroup = 0; + } + else if (worldRank == 1) + { + mpiGroup = 1; + } + else + { + mpiGroup = 2; + } + + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + size_t steps = 20; + + if (mpiGroup == 0) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"Verbose", "0"}}; + coupler(shape, start, count, steps, engineParams); + } + + if (mpiGroup == 1) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"Verbose", "0"}}; + gene(shape, start, shape, steps, engineParams); + } + + if (mpiGroup == 2) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"Verbose", "0"}}; + xgc(shape, start, count, steps, engineParams); + } + + MPI_Barrier(MPI_COMM_WORLD); } - if (mpiGroup == 1) { - shape = {(size_t)mpiSize, 10}; - start = {(size_t)mpiRank, 0}; - count = {1, 10}; - adios2::Params engineParams = {{"RendezvousAppCount", "2"}, - {"MaxStreamsPerApp", "4"}, - {"OpenTimeoutSecs", "3"}, - {"Verbose", "0"}}; - gene(shape, start, shape, steps, engineParams); + Dims start, count, shape; + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + + if (worldSize < 4) + { + return; + } + + if (worldRank == 0) + { + mpiGroup = 0; + } + else if (worldRank == 1) + { + mpiGroup = 1; + } + else + { + mpiGroup = 2; + } + + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + + size_t steps = 20; + + if (mpiGroup == 0) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"EngineMode", "naive"}, + {"Verbose", "0"}}; + coupler(shape, start, count, steps, engineParams); + } + + if (mpiGroup == 1) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"EngineMode", "naive"}, + {"Verbose", "0"}}; + gene(shape, start, shape, steps, engineParams); + } + + if (mpiGroup == 2) + { + shape = {(size_t)mpiSize, 10}; + start = {(size_t)mpiRank, 0}; + count = {1, 10}; + adios2::Params engineParams = {{"RendezvousAppCount", "2"}, + {"MaxStreamsPerApp", "4"}, + {"OpenTimeoutSecs", "3"}, + {"EngineMode", "naive"}, + {"Verbose", "0"}}; + xgc(shape, start, count, steps, engineParams); + } + + MPI_Barrier(MPI_COMM_WORLD); } - - if (mpiGroup == 2) - { - shape = {(size_t)mpiSize, 10}; - start = {(size_t)mpiRank, 0}; - count = {1, 10}; - adios2::Params engineParams = {{"RendezvousAppCount", "2"}, - {"MaxStreamsPerApp", "4"}, - {"OpenTimeoutSecs", "3"}, - {"Verbose", "0"}}; - xgc(shape, start, count, steps, engineParams); - } - - MPI_Barrier(MPI_COMM_WORLD); } int main(int argc, char **argv) diff --git a/testing/adios2/engine/ssc/TestSscZeroBlock.cpp b/testing/adios2/engine/ssc/TestSscZeroBlock.cpp new file mode 100644 index 0000000000..019c7e9e4b --- /dev/null +++ b/testing/adios2/engine/ssc/TestSscZeroBlock.cpp @@ -0,0 +1,312 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ + +#include "TestSscCommon.h" +#include +#include +#include +#include +#include + +using namespace adios2; +int mpiRank = 0; +int mpiSize = 1; +MPI_Comm mpiComm; + +class SscEngineTest : public ::testing::Test +{ +public: + SscEngineTest() = default; +}; + +void Writer(const Dims &shape, const Dims &start, const Dims &count, + const size_t steps, const adios2::Params &engineParams, + const std::string &name) +{ + size_t datasize = + std::accumulate(shape.begin(), shape.end(), static_cast(1), + std::multiplies()); + adios2::ADIOS adios(mpiComm); + adios2::IO io = adios.DeclareIO("Test"); + io.SetEngine("ssc"); + io.SetParameters(engineParams); + std::vector myChars(datasize); + std::vector myUChars(datasize); + std::vector myShorts(datasize); + std::vector myUShorts(datasize); + std::vector myInts(datasize); + std::vector myUInts(datasize); + std::vector myFloats(datasize); + std::vector myDoubles(datasize); + std::vector> myComplexes(datasize); + std::vector> myDComplexes(datasize); + auto bpChars = io.DefineVariable("bpChars", shape, start, shape); + auto bpUChars = + io.DefineVariable("bpUChars", shape, start, shape); + auto bpShorts = io.DefineVariable("bpShorts", shape, start, shape); + auto bpUShorts = + io.DefineVariable("bpUShorts", shape, start, shape); + auto bpInts = io.DefineVariable("bpInts", shape, start, shape); + auto bpUInts = + io.DefineVariable("bpUInts", shape, start, shape); + auto bpFloats = io.DefineVariable("bpFloats", shape, start, shape); + auto bpDoubles = + io.DefineVariable("bpDoubles", shape, start, shape); + auto bpComplexes = io.DefineVariable>( + "bpComplexes", shape, start, shape); + auto bpDComplexes = io.DefineVariable>( + "bpDComplexes", shape, start, shape); + auto scalarInt = io.DefineVariable("scalarInt"); + auto stringVar = io.DefineVariable("stringVar"); + io.DefineAttribute("AttInt", 110); + adios2::Engine engine = io.Open(name, adios2::Mode::Write); + engine.LockWriterDefinitions(); + for (size_t i = 0; i < steps; ++i) + { + engine.BeginStep(); + if (mpiRank == 0) + { + GenData(myChars, i, start, shape, shape); + GenData(myUChars, i, start, shape, shape); + GenData(myShorts, i, start, shape, shape); + GenData(myUShorts, i, start, shape, shape); + GenData(myInts, i, start, shape, shape); + GenData(myUInts, i, start, shape, shape); + GenData(myFloats, i, start, shape, shape); + GenData(myDoubles, i, start, shape, shape); + GenData(myComplexes, i, start, shape, shape); + GenData(myDComplexes, i, start, shape, shape); + engine.Put(bpChars, myChars.data(), adios2::Mode::Sync); + engine.Put(bpUChars, myUChars.data(), adios2::Mode::Sync); + engine.Put(bpShorts, myShorts.data(), adios2::Mode::Sync); + engine.Put(bpUShorts, myUShorts.data(), adios2::Mode::Sync); + engine.Put(bpInts, myInts.data(), adios2::Mode::Sync); + engine.Put(bpUInts, myUInts.data(), adios2::Mode::Sync); + engine.Put(bpFloats, myFloats.data(), adios2::Mode::Sync); + engine.Put(bpDoubles, myDoubles.data(), adios2::Mode::Sync); + engine.Put(bpComplexes, myComplexes.data(), adios2::Mode::Sync); + engine.Put(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync); + engine.Put(scalarInt, static_cast(i)); + std::string s = "sample string sample string sample string"; + engine.Put(stringVar, s); + } + engine.EndStep(); + } + engine.Close(); +} + +void Reader(const Dims &shape, const Dims &start, const Dims &count, + const size_t steps, const adios2::Params &engineParams, + const std::string &name) +{ + adios2::ADIOS adios(mpiComm); + adios2::IO io = adios.DeclareIO("Test"); + io.SetEngine("ssc"); + io.SetParameters(engineParams); + adios2::Engine engine = io.Open(name, adios2::Mode::Read); + + size_t datasize = + std::accumulate(count.begin(), count.end(), static_cast(1), + std::multiplies()); + std::vector myChars(datasize); + std::vector myUChars(datasize); + std::vector myShorts(datasize); + std::vector myUShorts(datasize); + std::vector myInts(datasize); + std::vector myUInts(datasize); + std::vector myFloats(datasize); + std::vector myDoubles(datasize); + std::vector> myComplexes(datasize); + std::vector> myDComplexes(datasize); + + engine.LockReaderSelections(); + + while (true) + { + adios2::StepStatus status = engine.BeginStep(StepMode::Read, 5); + if (status == adios2::StepStatus::OK) + { + auto scalarInt = io.InquireVariable("scalarInt"); + auto blocksInfo = + engine.BlocksInfo(scalarInt, engine.CurrentStep()); + + for (const auto &bi : blocksInfo) + { + ASSERT_EQ(bi.IsValue, true); + ASSERT_EQ(bi.Value, engine.CurrentStep()); + ASSERT_EQ(scalarInt.Min(), engine.CurrentStep()); + ASSERT_EQ(scalarInt.Max(), engine.CurrentStep()); + } + + const auto &vars = io.AvailableVariables(); + ASSERT_EQ(vars.size(), 12); + size_t currentStep = engine.CurrentStep(); + adios2::Variable bpChars = + io.InquireVariable("bpChars"); + adios2::Variable bpUChars = + io.InquireVariable("bpUChars"); + adios2::Variable bpShorts = + io.InquireVariable("bpShorts"); + adios2::Variable bpUShorts = + io.InquireVariable("bpUShorts"); + adios2::Variable bpInts = io.InquireVariable("bpInts"); + adios2::Variable bpUInts = + io.InquireVariable("bpUInts"); + adios2::Variable bpFloats = + io.InquireVariable("bpFloats"); + adios2::Variable bpDoubles = + io.InquireVariable("bpDoubles"); + adios2::Variable> bpComplexes = + io.InquireVariable>("bpComplexes"); + adios2::Variable> bpDComplexes = + io.InquireVariable>("bpDComplexes"); + adios2::Variable stringVar = + io.InquireVariable("stringVar"); + + bpChars.SetSelection({start, count}); + bpUChars.SetSelection({start, count}); + bpShorts.SetSelection({start, count}); + bpUShorts.SetSelection({start, count}); + bpInts.SetSelection({start, count}); + bpUInts.SetSelection({start, count}); + bpFloats.SetSelection({start, count}); + bpDoubles.SetSelection({start, count}); + bpComplexes.SetSelection({start, count}); + bpDComplexes.SetSelection({start, count}); + + engine.Get(bpChars, myChars.data(), adios2::Mode::Sync); + engine.Get(bpUChars, myUChars.data(), adios2::Mode::Sync); + engine.Get(bpShorts, myShorts.data(), adios2::Mode::Sync); + engine.Get(bpUShorts, myUShorts.data(), adios2::Mode::Sync); + engine.Get(bpInts, myInts.data(), adios2::Mode::Sync); + engine.Get(bpUInts, myUInts.data(), adios2::Mode::Sync); + + VerifyData(myChars.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myUChars.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myShorts.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myUShorts.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myInts.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myUInts.data(), currentStep, start, count, shape, + mpiRank); + + engine.Get(bpFloats, myFloats.data(), adios2::Mode::Deferred); + engine.Get(bpDoubles, myDoubles.data(), adios2::Mode::Deferred); + engine.Get(bpComplexes, myComplexes.data(), adios2::Mode::Deferred); + engine.Get(bpDComplexes, myDComplexes.data(), + adios2::Mode::Deferred); + engine.PerformGets(); + + VerifyData(myFloats.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myDoubles.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myComplexes.data(), currentStep, start, count, shape, + mpiRank); + VerifyData(myDComplexes.data(), currentStep, start, count, shape, + mpiRank); + + std::string s; + engine.Get(stringVar, s); + engine.PerformGets(); + ASSERT_EQ(s, "sample string sample string sample string"); + ASSERT_EQ(stringVar.Min(), + "sample string sample string sample string"); + ASSERT_EQ(stringVar.Max(), + "sample string sample string sample string"); + + int i; + engine.Get(scalarInt, &i); + engine.PerformGets(); + ASSERT_EQ(i, currentStep); + engine.EndStep(); + } + else if (status == adios2::StepStatus::EndOfStream) + { + std::cout << "[Rank " + std::to_string(mpiRank) + + "] SscTest reader end of stream!" + << std::endl; + break; + } + } + auto attInt = io.InquireAttribute("AttInt"); + std::cout << "[Rank " + std::to_string(mpiRank) + "] Attribute received " + << attInt.Data()[0] << ", expected 110" << std::endl; + ASSERT_EQ(110, attInt.Data()[0]); + ASSERT_NE(111, attInt.Data()[0]); + engine.Close(); +} + +TEST_F(SscEngineTest, TestSscZeroBlock) +{ + { + std::string filename = "TestSscZeroBlock"; + adios2::Params engineParams = {{"Verbose", "0"}}; + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + Dims shape = {10, (size_t)mpiSize * 2}; + Dims start = {2, (size_t)mpiRank * 2}; + Dims count = {5, 2}; + size_t steps = 100; + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + MPI_Barrier(MPI_COMM_WORLD); + } + { + std::string filename = "TestSscZeroBlock"; + adios2::Params engineParams = {{"Verbose", "0"}, + {"EngineMode", "naive"}}; + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + int mpiGroup = worldRank / (worldSize / 2); + MPI_Comm_split(MPI_COMM_WORLD, mpiGroup, worldRank, &mpiComm); + MPI_Comm_rank(mpiComm, &mpiRank); + MPI_Comm_size(mpiComm, &mpiSize); + Dims shape = {10, (size_t)mpiSize * 2}; + Dims start = {2, (size_t)mpiRank * 2}; + Dims count = {5, 2}; + size_t steps = 100; + if (mpiGroup == 0) + { + Writer(shape, start, count, steps, engineParams, filename); + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + if (mpiGroup == 1) + { + Reader(shape, start, count, steps, engineParams, filename); + } + MPI_Barrier(MPI_COMM_WORLD); + } +} + +int main(int argc, char **argv) +{ + MPI_Init(&argc, &argv); + int worldRank, worldSize; + MPI_Comm_rank(MPI_COMM_WORLD, &worldRank); + MPI_Comm_size(MPI_COMM_WORLD, &worldSize); + ::testing::InitGoogleTest(&argc, argv); + int result = RUN_ALL_TESTS(); + + MPI_Finalize(); + return result; +} diff --git a/testing/adios2/interface/TestADIOSDefineAttribute.cpp b/testing/adios2/interface/TestADIOSDefineAttribute.cpp index 3eb7aa1816..cab73fdcb2 100644 --- a/testing/adios2/interface/TestADIOSDefineAttribute.cpp +++ b/testing/adios2/interface/TestADIOSDefineAttribute.cpp @@ -156,6 +156,7 @@ TEST_F(ADIOSDefineAttributeTest, DefineAttributeTypeByValue) EXPECT_EQ(attributeS1.Data()[0], currentTestData.S1); EXPECT_EQ(attributeS1.Data().size(), 1); EXPECT_EQ(attributeS1.Type(), "string"); + EXPECT_TRUE(attributeS1.IsValue()); ASSERT_EQ(attributeI8.Data().size() == 1, true); ASSERT_EQ(attributeI8.Data().empty(), false); @@ -163,6 +164,7 @@ TEST_F(ADIOSDefineAttributeTest, DefineAttributeTypeByValue) EXPECT_EQ(attributeI8.Data()[0], currentTestData.I8.front()); EXPECT_EQ(attributeI8.Data().size(), 1); EXPECT_EQ(attributeI8.Type(), "int8_t"); + EXPECT_TRUE(attributeI8.IsValue()); ASSERT_EQ(attributeI16.Data().size() == 1, true); ASSERT_EQ(attributeI16.Data().empty(), false); @@ -170,6 +172,7 @@ TEST_F(ADIOSDefineAttributeTest, DefineAttributeTypeByValue) EXPECT_EQ(attributeI16.Data()[0], currentTestData.I16.front()); EXPECT_EQ(attributeI16.Data().size(), 1); EXPECT_EQ(attributeI16.Type(), "int16_t"); + EXPECT_TRUE(attributeI16.IsValue()); ASSERT_EQ(attributeI32.Data().size() == 1, true); ASSERT_EQ(attributeI32.Data().empty(), false); @@ -177,6 +180,7 @@ TEST_F(ADIOSDefineAttributeTest, DefineAttributeTypeByValue) EXPECT_EQ(attributeI32.Data()[0], currentTestData.I32.front()); EXPECT_EQ(attributeI32.Data().size(), 1); EXPECT_EQ(attributeI32.Type(), "int32_t"); + EXPECT_TRUE(attributeI32.IsValue()); ASSERT_EQ(attributeI64.Data().size() == 1, true); ASSERT_EQ(attributeI64.Data().empty(), false); @@ -184,6 +188,7 @@ TEST_F(ADIOSDefineAttributeTest, DefineAttributeTypeByValue) EXPECT_EQ(attributeI64.Data()[0], currentTestData.I64.front()); EXPECT_EQ(attributeI64.Data().size(), 1); EXPECT_EQ(attributeI64.Type(), "int64_t"); + EXPECT_TRUE(attributeI64.IsValue()); EXPECT_EQ(sizeof(attributeI64.Data()[0]), 8); ASSERT_EQ(attributeU8.Data().size() == 1, true); @@ -192,6 +197,7 @@ TEST_F(ADIOSDefineAttributeTest, DefineAttributeTypeByValue) EXPECT_EQ(attributeU8.Data()[0], currentTestData.U8.front()); EXPECT_EQ(attributeU8.Data().size(), 1); EXPECT_EQ(attributeU8.Type(), "uint8_t"); + EXPECT_TRUE(attributeU8.IsValue()); ASSERT_EQ(attributeU16.Data().size() == 1, true); ASSERT_EQ(attributeU16.Data().empty(), false); @@ -199,6 +205,7 @@ TEST_F(ADIOSDefineAttributeTest, DefineAttributeTypeByValue) EXPECT_EQ(attributeU16.Data()[0], currentTestData.U16.front()); EXPECT_EQ(attributeU16.Data().size(), 1); EXPECT_EQ(attributeU16.Type(), "uint16_t"); + EXPECT_TRUE(attributeU16.IsValue()); ASSERT_EQ(attributeU32.Data().size() == 1, true); ASSERT_EQ(attributeU32.Data().empty(), false); @@ -206,6 +213,7 @@ TEST_F(ADIOSDefineAttributeTest, DefineAttributeTypeByValue) EXPECT_EQ(attributeU32.Data()[0], currentTestData.U32.front()); EXPECT_EQ(attributeU32.Data().size(), 1); EXPECT_EQ(attributeU32.Type(), "uint32_t"); + EXPECT_TRUE(attributeU32.IsValue()); ASSERT_EQ(attributeU64.Data().size() == 1, true); ASSERT_EQ(attributeU64.Data().empty(), false); @@ -213,6 +221,7 @@ TEST_F(ADIOSDefineAttributeTest, DefineAttributeTypeByValue) EXPECT_EQ(attributeU64.Data()[0], currentTestData.U64.front()); EXPECT_EQ(attributeU64.Data().size(), 1); EXPECT_EQ(attributeU64.Type(), "uint64_t"); + EXPECT_TRUE(attributeU64.IsValue()); EXPECT_EQ(sizeof(attributeU64.Data()[0]), 8); ASSERT_EQ(attributeFloat.Data().size() == 1, true); @@ -221,6 +230,7 @@ TEST_F(ADIOSDefineAttributeTest, DefineAttributeTypeByValue) EXPECT_EQ(attributeFloat.Data()[0], currentTestData.R32.front()); EXPECT_EQ(attributeFloat.Data().size(), 1); EXPECT_EQ(attributeFloat.Type(), "float"); + EXPECT_TRUE(attributeFloat.IsValue()); ASSERT_EQ(attributeDouble.Data().size() == 1, true); ASSERT_EQ(attributeDouble.Data().empty(), false); @@ -228,6 +238,7 @@ TEST_F(ADIOSDefineAttributeTest, DefineAttributeTypeByValue) EXPECT_EQ(attributeDouble.Data()[0], currentTestData.R64.front()); EXPECT_EQ(attributeDouble.Data().size(), 1); EXPECT_EQ(attributeDouble.Type(), "double"); + EXPECT_TRUE(attributeDouble.IsValue()); } TEST_F(ADIOSDefineAttributeTest, DefineAttributeTypeByReference) @@ -483,30 +494,35 @@ TEST_F(ADIOSDefineAttributeTest, GetAttribute) EXPECT_EQ(attributeS3.Name(), s3_Single); EXPECT_EQ(attributeS3.Data().size(), 3); EXPECT_EQ(attributeS3.Type(), "string"); + EXPECT_FALSE(attributeS3.IsValue()); ASSERT_EQ(attributeI8.Data().size() == 1, false); ASSERT_EQ(attributeI8.Data().empty(), false); EXPECT_EQ(attributeI8.Name(), i8_Single); EXPECT_EQ(attributeI8.Data().size(), numberOfElements); EXPECT_EQ(attributeI8.Type(), "int8_t"); + EXPECT_FALSE(attributeI8.IsValue()); ASSERT_EQ(attributeI16.Data().size() == 1, false); ASSERT_EQ(attributeI16.Data().empty(), false); EXPECT_EQ(attributeI16.Name(), i16_Single); EXPECT_EQ(attributeI16.Data().size(), numberOfElements); EXPECT_EQ(attributeI16.Type(), "int16_t"); + EXPECT_FALSE(attributeI16.IsValue()); ASSERT_EQ(attributeI32.Data().size() == 1, false); ASSERT_EQ(attributeI32.Data().empty(), false); EXPECT_EQ(attributeI32.Name(), i32_Single); EXPECT_EQ(attributeI32.Data().size(), numberOfElements); EXPECT_EQ(attributeI32.Type(), "int32_t"); + EXPECT_FALSE(attributeI32.IsValue()); ASSERT_EQ(attributeI64.Data().size() == 1, false); ASSERT_EQ(attributeI64.Data().empty(), false); EXPECT_EQ(attributeI64.Name(), i64_Single); EXPECT_EQ(attributeI64.Data().size(), numberOfElements); EXPECT_EQ(attributeI64.Type(), "int64_t"); + EXPECT_FALSE(attributeI64.IsValue()); EXPECT_EQ(sizeof(attributeI64.Data()[0]), 8); ASSERT_EQ(attributeU8.Data().size() == 1, false); @@ -514,24 +530,28 @@ TEST_F(ADIOSDefineAttributeTest, GetAttribute) EXPECT_EQ(attributeU8.Name(), u8_Single); EXPECT_EQ(attributeU8.Data().size(), numberOfElements); EXPECT_EQ(attributeU8.Type(), "uint8_t"); + EXPECT_FALSE(attributeU8.IsValue()); ASSERT_EQ(attributeU16.Data().size() == 1, false); ASSERT_EQ(attributeU16.Data().empty(), false); EXPECT_EQ(attributeU16.Name(), u16_Single); EXPECT_EQ(attributeU16.Data().size(), numberOfElements); EXPECT_EQ(attributeU16.Type(), "uint16_t"); + EXPECT_FALSE(attributeU16.IsValue()); ASSERT_EQ(attributeU32.Data().size() == 1, false); ASSERT_EQ(attributeU32.Data().empty(), false); EXPECT_EQ(attributeU32.Name(), u32_Single); EXPECT_EQ(attributeU32.Data().size(), numberOfElements); EXPECT_EQ(attributeU32.Type(), "uint32_t"); + EXPECT_FALSE(attributeU32.IsValue()); ASSERT_EQ(attributeU64.Data().size() == 1, false); ASSERT_EQ(attributeU64.Data().empty(), false); EXPECT_EQ(attributeU64.Name(), u64_Single); EXPECT_EQ(attributeU64.Data().size(), numberOfElements); EXPECT_EQ(attributeU64.Type(), "uint64_t"); + EXPECT_FALSE(attributeU64.IsValue()); EXPECT_EQ(sizeof(attributeU64.Data()[0]), 8); ASSERT_EQ(attributeFloat.Data().size() == 1, false); @@ -539,12 +559,14 @@ TEST_F(ADIOSDefineAttributeTest, GetAttribute) EXPECT_EQ(attributeFloat.Name(), float_Single); EXPECT_EQ(attributeFloat.Data().size(), numberOfElements); EXPECT_EQ(attributeFloat.Type(), "float"); + EXPECT_FALSE(attributeFloat.IsValue()); ASSERT_EQ(attributeDouble.Data().size() == 1, false); ASSERT_EQ(attributeDouble.Data().empty(), false); EXPECT_EQ(attributeDouble.Name(), double_Single); EXPECT_EQ(attributeDouble.Data().size(), numberOfElements); EXPECT_EQ(attributeDouble.Type(), "double"); + EXPECT_FALSE(attributeDouble.IsValue()); // Verify data for (size_t index = 0; index < numberOfElements; index++) diff --git a/thirdparty/dill/dill/.github/workflows/build-and-test.yml b/thirdparty/dill/dill/.github/workflows/build-and-test.yml index b05345f552..ce9fd1cd3e 100644 --- a/thirdparty/dill/dill/.github/workflows/build-and-test.yml +++ b/thirdparty/dill/dill/.github/workflows/build-and-test.yml @@ -1,31 +1,115 @@ -on: - push: - branches: - - master - pull_request: - branches: - - master +name: Build and Test + +on: + push: + branches: + - master + pull_request: + branches: + - master jobs: - ci: - runs-on: ${{ matrix.os-image }} + linux: + # The jobs should run pretty quick; anything over 30m essentially means + # someting is stuck somewhere + timeout-minutes: 30 + runs-on: ubuntu-latest + container: ${{ matrix.container }} + env: + GH_YML_JOBNAME: ${{ matrix.os }}-${{ matrix.compiler }} + GH_YML_BUILDTYPE: ${{ matrix.buildtype }} + GH_YML_SHA: ${{ github.event.pull_request.head.sha || github.sha }} strategy: fail-fast: false matrix: - os-image: [ubuntu-latest, macos-latest] + buildtype: [ release, debug ] + os: [ centos7, centos8, ubuntu1604, ubuntu1804, ubuntu2004 ] + compiler: [ clang, gcc, nvhpc ] + exclude: + - { os: centos8, compiler: nvhpc } + - { os: ubuntu1604, compiler: nvhpc } + - { os: ubuntu1804, compiler: nvhpc } + include: + - os: centos7 + container: centos:7 + - os: centos8 + container: centos:8 + - os: ubuntu1604 + container: ubuntu:16.04 + - os: ubuntu1804 + container: ubuntu:18.04 + - os: ubuntu2004 + container: ubuntu:20.04 + - os: centos7 + compiler: nvhpc + container: nvcr.io/nvidia/nvhpc:21.2-devel-cuda11.2-centos7 + - os: ubuntu2004 + compiler: nvhpc + container: nvcr.io/nvidia/nvhpc:21.2-devel-cuda11.2-ubuntu20.04 steps: - uses: actions/checkout@v2 with: ref: ${{ github.event.pull_request.head.sha }} + path: source + - name: Setup + run: source/scripts/ci/setup/linux.sh + - name: Update + run: source/scripts/ci/gh-actions/run.sh update + - name: Configure + run: source/scripts/ci/gh-actions/run.sh configure + - name: Build + run: source/scripts/ci/gh-actions/run.sh build + - name: Test + run: source/scripts/ci/gh-actions/run.sh test + + mac_and_windows: + # The jobs should run pretty quick; anything over 30m essentially means + # someting is stuck somewhere + timeout-minutes: 30 + runs-on: ${{ matrix.vm }} + env: + GH_YML_JOBNAME: ${{ matrix.jobname }} + GH_YML_BUILDTYPE: ${{ matrix.buildtype }} + GH_YML_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + + strategy: + fail-fast: false + matrix: + buildtype: [ release, debug ] + jobname: [ + # windows-vs2019-msvc, + windows-vs2019-clang, + macos-clang ] + include: + #- jobname: windows-vs2019-msvc + # vm: windows-latest + - jobname: windows-vs2019-clang + vm: windows-latest + - jobname: macos-clang + vm: macos-latest + + defaults: + run: + shell: bash + + steps: + - uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + path: source + - name: Setup + if: ${{ runner.os == 'Windows' }} + run: source/scripts/ci/setup/windows.sh - name: Setup - run: ci/gh-actions/setup.sh + if: ${{ runner.os == 'macOS' }} + run: source/scripts/ci/setup/macos.sh + - name: Update + run: source/scripts/ci/gh-actions/run.sh update - name: Configure - run: ci/gh-actions/run.sh configure + run: source/scripts/ci/gh-actions/run.sh configure - name: Build - run: ci/gh-actions/run.sh build + run: source/scripts/ci/gh-actions/run.sh build - name: Test - run: ci/gh-actions/run.sh test - - name: Install - run: ci/gh-actions/run.sh install + run: source/scripts/ci/gh-actions/run.sh test diff --git a/thirdparty/dill/dill/.github/workflows/triggers.yml b/thirdparty/dill/dill/.github/workflows/triggers.yml new file mode 100644 index 0000000000..3ee5556d9e --- /dev/null +++ b/thirdparty/dill/dill/.github/workflows/triggers.yml @@ -0,0 +1,15 @@ +name: Triggers + +on: + workflow_run: + workflows: ["Build and Test"] + types: [requested] + +jobs: + all_triggers: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Post CDash Status + run: scripts/ci/scripts/post-cdash-status.sh ${{ github.event.repository.full_name }} ${{ github.event.workflow_run.head_sha }} ${{ secrets.GITHUB_TOKEN }} diff --git a/thirdparty/dill/dill/CMakeLists.txt b/thirdparty/dill/dill/CMakeLists.txt index 7350196dd3..db81834392 100644 --- a/thirdparty/dill/dill/CMakeLists.txt +++ b/thirdparty/dill/dill/CMakeLists.txt @@ -1,5 +1,8 @@ cmake_minimum_required(VERSION 3.0) +# The directory label is used for CDash to treat DILL as a subproject of GTKorvo +set(CMAKE_DIRECTORY_LABELS DILL) + project(DILL VERSION 2.4.1 LANGUAGES C CXX) # Some boilerplate to setup nice output directories @@ -36,6 +39,23 @@ if(NOT DEFINED DILL_HEADER_COMPONENT) set(DILL_HEADER_COMPONENT dev) endif() +if(WIN32) + # Automagic to do the DLL / LIB song and dance + set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS TRUE) + + # Silence MSVC warnings + if(CMAKE_C_COMPILER_ID MATCHES "MSVC" OR + CMAKE_C_SIMULATE_ID MATCHES "MSVC") + add_definitions( + -D_CRT_SECURE_NO_DEPRECATE + -D_CRT_SECURE_NO_WARNINGS + -D_SCL_SECURE_NO_DEPRECATE + -D_WINSOCK_DEPRECATED_NO_WARNINGS + -D_CRT_NONSTDC_NO_DEPRECATE) + set (MSVC_PERL_FLAGS "-msvc-long") + endif() +endif() + # Setup shared library defaults. If explicitly specified somehow, then default # to that. Otherwise base the default on whether or not shared libs are even # supported. @@ -84,6 +104,9 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "i.86|x86_64|AMD64") set(TEST_PERL_FLAGS -no_float -max_arg=2) set(ARCH_FILE x86) endif() + if(NOT (CMAKE_SIZEOF_VOID_P EQUAL SIZEOF_LONG)) + set(BASE_OPS_ARG "-msvc_long") + endif() elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "arm.5.*") set(NATIVE_ARCH arm5) set(HOST_ARM5 1) @@ -119,6 +142,8 @@ endif() if(CMAKE_SYSTEM_NAME MATCHES "Darwin|Linux") set(USE_MMAP_CODE_SEG 1) +elseif(CMAKE_SYSTEM_NAME MATCHES "Windows") + set(USE_VIRTUAL_PROTECT 1) endif() include(TestBigEndian) @@ -221,6 +246,8 @@ else() set(EMULATION_POSSIBLE FALSE) endif() +find_package(Perl REQUIRED) + option(DILL_ENABLE_DISASSEMBLY "enable binutils-based disassembly (default is OFF)" OFF) @@ -228,7 +255,7 @@ set(ARCHITECTURES sparc ppc64le virtual x86 x86_64 ia64 arm5 arm6 arm8) foreach(_arch ${ARCHITECTURES}) add_custom_command( OUTPUT dill_${_arch}.c - COMMAND perl ${CMAKE_CURRENT_SOURCE_DIR}/${_arch}.ops + COMMAND ${PERL_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/${_arch}.ops DEPENDS ${_arch}.ops ) list(APPEND arch_files dill_${_arch}.c ${_arch}.c) @@ -239,7 +266,7 @@ endforeach() add_custom_command( OUTPUT dill.h dill.c - COMMAND perl ${CMAKE_CURRENT_SOURCE_DIR}/base.ops + COMMAND ${PERL_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/base.ops ${BASE_OPS_ARG} DEPENDS base.ops ) @@ -293,6 +320,7 @@ check_include_files(unistd.h HAVE_UNISTD_H) check_include_files(stdarg.h STDC_HEADERS) check_include_files(malloc.h HAVE_MALLOC_H) check_include_files(memory.h HAVE_MEMORY_H) +check_include_files(sys/mman.h HAVE_SYS_MMAN_H) include(CheckSymbolExists) check_symbol_exists(__clear_cache "" CLEAR_CACHE_DEFINED) message(STATUS "Clear cache defined is ${CLEAR_CACHE_DEFINED}") @@ -473,6 +501,12 @@ if(BUILD_TESTING) add_subdirectory(vtests) endif() +configure_file( + ${PROJECT_SOURCE_DIR}/CTestCustom.cmake.in + ${PROJECT_BINARY_DIR}/CTestCustom.cmake + @ONLY +) + option(DILL_QUIET "Suppress summary output" OFF) if(NOT DILL_QUIET) message(STATUS) diff --git a/thirdparty/dill/dill/CTestConfig.cmake b/thirdparty/dill/dill/CTestConfig.cmake new file mode 100644 index 0000000000..3416157733 --- /dev/null +++ b/thirdparty/dill/dill/CTestConfig.cmake @@ -0,0 +1,9 @@ +set(CTEST_PROJECT_NAME "GTKorvo") +set(CTEST_NIGHTLY_START_TIME "01:00:00 UTC") + +set(CTEST_DROP_METHOD "http") +set(CTEST_DROP_SITE "open.cdash.org") +set(CTEST_DROP_LOCATION "/submit.php?project=GTKorvo") +set(CTEST_DROP_SITE_CDASH TRUE) + +set(CTEST_LABELS_FOR_SUBPROJECTS DILL) diff --git a/thirdparty/dill/dill/CTestCustom.cmake.in b/thirdparty/dill/dill/CTestCustom.cmake.in new file mode 100644 index 0000000000..dc8ba0c5a5 --- /dev/null +++ b/thirdparty/dill/dill/CTestCustom.cmake.in @@ -0,0 +1,13 @@ +set(CTEST_CUSTOM_MAXIMUM_NUMBER_OF_ERRORS 1000) +set(CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS 1000) +set(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE 1048576) +set(CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE 1048576) + +list(APPEND CTEST_CUSTOM_WARNING_EXCEPTION +) + +list(APPEND CTEST_CUSTOM_COVERAGE_EXCLUDE +) + +list(APPEND CTEST_CUSTOM_TESTS_IGNORE +) diff --git a/thirdparty/dill/dill/CTestCustom.ctest.in b/thirdparty/dill/dill/CTestCustom.ctest.in deleted file mode 100644 index fbdf009495..0000000000 --- a/thirdparty/dill/dill/CTestCustom.ctest.in +++ /dev/null @@ -1,6 +0,0 @@ -set (CTEST_CUSTOM_MEMCHECK_IGNORE) -set (CTEST_SVN_UPDATE_OPTIONS --password anon --username anon) -set (CTEST_SVN_OPTIONS --password anon --username anon) -set (SVN_UPDATE_OPTIONS --password anon --username anon) -set (SVN_OPTIONS --password anon --username anon) - diff --git a/thirdparty/dill/dill/base.ops b/thirdparty/dill/dill/base.ops index b806575df8..a31e033429 100755 --- a/thirdparty/dill/dill/base.ops +++ b/thirdparty/dill/dill/base.ops @@ -1,5 +1,14 @@ #!perl + +$MSVC_LONG = 0; + +while ($_ = $ARGV[0]) { + shift; + last if /^--$/; + if (/^-msvc_long/) {$MSVC_LONG = 1;} +} + sub upperc { local($_) = pop(@_); tr/[a-z]/[A-Z]/; @@ -85,62 +94,62 @@ print HOUT<j->ret)(s, DILL_" . &upperc($dill_type) . ", 0, src)\n"; if ($_ eq 'p') { - print HOUT "#define dill_ret${type}i(s, imm) (s->j->reti)(s, DILL_" . &upperc($type) . ", 0, (unsigned long) imm)\n"; + print HOUT "#define dill_ret${type}i(s, imm) (s->j->reti)(s, DILL_" . &upperc($type) . ", 0, (IMM_TYPE) imm)\n"; } elsif (($_ eq 'f') || ($_ eq 'd')) { print HOUT "#define dill_ret${type}i(s, imm) (s->j->retf)(s, DILL_" . &upperc($type) . ", 0, imm)\n"; } else { @@ -443,6 +452,17 @@ extern "C" { #endif +EOF +if ($MSVC_LONG) { +print HOUT "#include \n"; +print HOUT "#define IMM_TYPE intptr_t\n"; +print HOUT "#define UIMM_TYPE uintptr_t\n"; +} else { +print HOUT "#define IMM_TYPE long\n"; +print HOUT "#define UIMM_TYPE unsigned long\n"; +} +print HOUT<j->type_align[t] #define dill_type_size(s, t) s->j->type_size[t] @@ -731,8 +741,8 @@ enum { DILL_US, /* unsigned short */ DILL_I, /* int */ DILL_U, /* unsigned */ - DILL_L, /* long */ - DILL_UL, /* unsigned long */ + DILL_L, /* long (full register size */ + DILL_UL, /* unsigned long (full register size) */ DILL_P, /* pointer */ DILL_F, /* floating */ DILL_D, /* double */ diff --git a/thirdparty/dill/dill/ci/gh-actions/run.sh b/thirdparty/dill/dill/ci/gh-actions/run.sh deleted file mode 100755 index a02a085ee6..0000000000 --- a/thirdparty/dill/dill/ci/gh-actions/run.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -export CI_ROOT_DIR="${GITHUB_WORKSPACE//\\//}/.." -export CI_SOURCE_DIR="${GITHUB_WORKSPACE//\\//}" -export CI_DEP_DIR="${CI_ROOT_DIR}/dependencies" -export CI_BIN_DIR="${CI_ROOT_DIR}/build" - -export CMAKE_PREFIX_PATH=${CI_DEP_DIR}/install -export PATH=${CI_DEP_DIR}/tools/bin:${CI_DEP_DIR}/install/bin:${PATH} -case "$(uname -s)" in - Linux) - export LD_LIBRARY_PATH=${CI_DEP_DIR}/install/lib:${LD_LIBRARY_PATH} - ;; - Darwin) - export DYLD_LIBRARY_PATH=${CI_DEP_DIR}/install/lib:${DYLD_LIBRARY_PATH} - ;; -esac - - -mkdir -p ${CI_BIN_DIR} -cd ${CI_BIN_DIR} - -case "$1" in - configure) - cmake -GNinja -DCMAKE_INSTALL_PREFIX=${CI_ROOT_DIR}/install ${CI_SOURCE_DIR} - ;; - build) - ninja - ;; - test) - if [ "$(uname -s)" = "Darwin" ] - then - # Disable the firewall - sudo /usr/libexec/ApplicationFirewall/socketfilterfw --setglobalstate off - - # Force the use of the loopback interface - #export CM_IP="127.0.0.1" - export CM_HOSTNAME="localhost" - CTEST_EXCLUDES="mtests_non_blocking_bulk" - fi - ctest --timeout 300 -j2 -VV -E "${CTEST_EXCLUDES}" - ;; - install) - ninja install - ;; -esac diff --git a/thirdparty/dill/dill/ci/gh-actions/setup-linux.sh b/thirdparty/dill/dill/ci/gh-actions/setup-linux.sh deleted file mode 100755 index 7f2323a6cc..0000000000 --- a/thirdparty/dill/dill/ci/gh-actions/setup-linux.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -echo -echo "Installing ninja" -mkdir -p ${CI_DEP_DIR}/tools/bin -cd ${CI_DEP_DIR}/tools/bin -curl -O -L https://github.com/ninja-build/ninja/releases/download/v1.10.0/ninja-linux.zip -unzip ninja-linux.zip - -# Install cmake -echo -echo "Installing CMake" -mkdir -p ${CI_DEP_DIR}/tools -cd ${CI_DEP_DIR}/tools -curl -L https://github.com/Kitware/CMake/releases/download/v3.3.2/cmake-3.3.2-Linux-x86_64.tar.gz | tar --strip-components=1 -xz - -export PATH=${CI_DEP_DIR}/tools/bin:${PATH} diff --git a/thirdparty/dill/dill/ci/gh-actions/setup-macos.sh b/thirdparty/dill/dill/ci/gh-actions/setup-macos.sh deleted file mode 100755 index f38950185f..0000000000 --- a/thirdparty/dill/dill/ci/gh-actions/setup-macos.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -echo -echo "Installing ninja" -brew install ninja - -# Install cmake -echo -echo "Installing CMake" -mkdir -p ${CI_DEP_DIR}/tools -cd ${CI_DEP_DIR}/tools -curl -L https://github.com/Kitware/CMake/releases/download/v3.3.2/cmake-3.3.2-Darwin-x86_64.tar.gz | tar --strip-components=3 -xz - -export PATH=${CI_DEP_DIR}/tools/bin:${PATH} diff --git a/thirdparty/dill/dill/ci/gh-actions/setup.sh b/thirdparty/dill/dill/ci/gh-actions/setup.sh deleted file mode 100755 index 04699eee36..0000000000 --- a/thirdparty/dill/dill/ci/gh-actions/setup.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -export CI_ROOT_DIR="${GITHUB_WORKSPACE//\\//}/.." -export CI_SOURCE_DIR="${GITHUB_WORKSPACE//\\//}" -export CI_DEP_DIR="${CI_ROOT_DIR}/dependencies" - -# Install ninja, pkgconfig, and libfabric -case "$(uname -s)" in - Linux) - . $(dirname ${BASH_SOURCE[0]})/setup-linux.sh - ;; - Darwin) - . $(dirname ${BASH_SOURCE[0]})/setup-macos.sh - ;; -esac - -export CMAKE_PREFIX_PATH=${CI_DEP_DIR}/install - diff --git a/thirdparty/dill/dill/config.h.cmake b/thirdparty/dill/dill/config.h.cmake index a9622f565d..f04719721b 100644 --- a/thirdparty/dill/dill/config.h.cmake +++ b/thirdparty/dill/dill/config.h.cmake @@ -1,6 +1,6 @@ /* config.h.in. Generated from configure.ac by autoheader. */ -/* Define with the build version of atl */ +/* Define with the build version of dill */ #cmakedefine DILL_VERSION "@DILL_VERSION@" /* Define if bfd functions use bfd_byte arguments. */ @@ -27,6 +27,9 @@ /* Define to 1 if you have the header file. */ #cmakedefine HAVE_MALLOC_H +/* Define to 1 if you have the header file. */ +#cmakedefine HAVE_SYS_MMAN_H + /* Define to 1 if you have the header file. */ #cmakedefine HAVE_MEMORY_H @@ -143,6 +146,9 @@ /* Define this is mmap should be used instead of malloc() for code memory */ #cmakedefine USE_MMAP_CODE_SEG +/* Define this is VirtualProtect should be used to change memory protections */ +#cmakedefine USE_VIRTUAL_PROTECT + /* Define if byteorder is bigendian */ #cmakedefine WORDS_BIGENDIAN diff --git a/thirdparty/dill/dill/dill_internal.h b/thirdparty/dill/dill/dill_internal.h index 205c77c368..effb408911 100644 --- a/thirdparty/dill/dill/dill_internal.h +++ b/thirdparty/dill/dill/dill_internal.h @@ -8,8 +8,8 @@ typedef struct arg_info { char type; char is_register; /* true if parameter is in register */ char is_immediate; /* true if actual is an immediate */ - unsigned in_reg; /* callee register it's in */ - unsigned out_reg; /* caller register it's in */ + int in_reg; /* callee register it's in */ + int out_reg; /* caller register it's in */ int offset; /* otherwise at this offset from v_pp */ int used; } *arg_info_list; @@ -242,7 +242,7 @@ extern void dill_mark_call_location(dill_stream s, const char *xfer_name, void *xfer_address); extern void dill_mark_ret_location(dill_stream s); extern void dill_end_vararg_push(dill_stream s); -EXTERN void dill_dump_reg(dill_stream s, int typ, int reg); +extern void dill_dump_reg(dill_stream s, int typ, int reg); extern void setup_VM_proc(dill_stream s); typedef struct dill_pkg_1 { diff --git a/thirdparty/dill/dill/dill_pkg.c b/thirdparty/dill/dill/dill_pkg.c index 38edda1b26..63bffa0010 100644 --- a/thirdparty/dill/dill/dill_pkg.c +++ b/thirdparty/dill/dill/dill_pkg.c @@ -29,17 +29,17 @@ unpack_package(char *package, call_t *t, char **code_p) t->call_locs = malloc(sizeof(t->call_locs[0]) * pkg->symbol_count); memset(t->call_locs, 0, sizeof(t->call_locs[0]) * pkg->symbol_count); for (count = 0; countsymbol_count; count++) { - int call_len; + size_t call_len; t->call_locs[count].loc = *((int*)p); t->call_locs[count].xfer_name = (p + sizeof(int)); call_len = sizeof(int) + strlen(t->call_locs[count].xfer_name) + 1; - call_len = (call_len + 7) & -8; /* round up to mod 8 */ + call_len = (call_len + 7) & (size_t)-8; /* round up to mod 8 */ p += call_len; } *code_p = p; } -EXTERN void* +extern void* dill_package_entry(char* package) { struct dill_pkg_1 *pkg = (struct dill_pkg_1 *) package; @@ -71,7 +71,7 @@ dill_lookup_xfer_addrs(call_t *t, xfer_entry *x) } } -EXTERN dill_exec_handle +extern dill_exec_handle dill_package_stitch(char *pkg, dill_extern_entry* extra_externs) { dill_exec_handle handle = malloc(sizeof(*handle)); diff --git a/thirdparty/dill/dill/dill_util.c b/thirdparty/dill/dill/dill_util.c index 07186e7fe2..e50c302fa0 100644 --- a/thirdparty/dill/dill/dill_util.c +++ b/thirdparty/dill/dill/dill_util.c @@ -24,6 +24,7 @@ #include #endif #include +#include #ifdef HAVE_UNISTD_H #include #endif @@ -218,7 +219,7 @@ set_mach_reset(dill_stream s, char *arch) return 0; } -EXTERN void +extern void dill_free_stream(dill_stream s) { if (s->p->branch_table.label_locs) free(s->p->branch_table.label_locs); @@ -256,7 +257,7 @@ dill_free_stream(dill_stream s) extern void DILLprint_version(); -EXTERN dill_stream +extern dill_stream dill_cross_init(char *arch) { dill_stream s = (dill_stream) malloc(sizeof(struct dill_stream_s)); @@ -325,7 +326,7 @@ extern void dill_virtual_init(dill_stream s); -EXTERN dill_stream +extern dill_stream dill_create_stream() { dill_stream s; @@ -350,11 +351,11 @@ dill_create_stream() #ifndef EMULATION_ONLY -EXTERN void +extern void dill_native_dcg(){} #endif -EXTERN dill_stream +extern dill_stream dill_create_raw_stream() { return dill_cross_init(NATIVE_ARCH); @@ -389,13 +390,13 @@ extend_params(dill_stream s, int argno) s->p->c_param_count = (argno + 1); } -EXTERN dill_reg +extern dill_reg dill_vparam(dill_stream s, int param_no) { return param_no; } -EXTERN int +extern int dill_param_reg(dill_stream s, int argno) { if (argno >= s->p->c_param_count) { @@ -406,7 +407,7 @@ dill_param_reg(dill_stream s, int argno) return s->p->c_param_args[argno].in_reg; } -EXTERN void +extern void dill_param_alloc(dill_stream s, int argno, int type, dill_reg *reg_p) { extend_params(s, argno); @@ -414,7 +415,7 @@ dill_param_alloc(dill_stream s, int argno, int type, dill_reg *reg_p) s->p->c_param_args[argno].type = type; } -EXTERN void +extern void dill_param_struct_alloc(dill_stream s, int argno, int type, dill_parameter_type *struct_p) { @@ -423,7 +424,7 @@ dill_param_struct_alloc(dill_stream s, int argno, int type, s->p->c_param_args[argno].type = type; } -EXTERN void +extern void dill_start_simple_proc(dill_stream s, const char *subr_name, int ret_type) { int i; @@ -458,14 +459,14 @@ dill_start_simple_proc(dill_stream s, const char *subr_name, int ret_type) } } -EXTERN void * +extern void * dill_get_fp(dill_exec_handle h) { return (void *) h->fp; } -EXTERN dill_exec_handle +extern dill_exec_handle dill_finalize(dill_stream s) { dill_exec_handle handle = malloc(sizeof(*handle)); @@ -479,16 +480,16 @@ dill_finalize(dill_stream s) return handle; } -EXTERN dill_exec_handle +extern dill_exec_handle dill_get_handle(dill_stream s) { char* native_base = s->p->native.code_base; dill_exec_handle handle = malloc(sizeof(*handle)); - int size = (long)s->p->native.code_limit - (long)s->p->native.code_base + END_OF_CODE_BUFFER; + intptr_t size = (intptr_t)s->p->native.code_limit - (intptr_t)s->p->native.code_base + END_OF_CODE_BUFFER; s->p->native.code_base = NULL; if (native_base == 0) { native_base = s->p->code_base; - size = (long)s->p->code_limit - (long)s->p->code_base + END_OF_CODE_BUFFER; + size = (intptr_t)s->p->code_limit - (intptr_t)s->p->code_base + END_OF_CODE_BUFFER; s->p->code_base = NULL; } handle->fp = (void(*)())s->p->fp; @@ -505,7 +506,7 @@ dill_get_handle(dill_stream s) return handle; } -EXTERN void +extern void dill_free_handle(dill_exec_handle handle) { handle->ref_count--; @@ -527,7 +528,7 @@ dill_free_handle(dill_exec_handle handle) free(handle); } -EXTERN void +extern void dill_ref_handle(dill_exec_handle handle) { handle->ref_count++; @@ -565,11 +566,11 @@ dill_build_package(dill_stream s, int *pkg_len) memcpy(pkg + pkg_size, s->p->code_base, dill_code_size(s)); pkg_size += dill_code_size(s); *pkg_len = pkg_size; - ((struct dill_pkg_1 *)pkg)->entry_offset = (char*)s->p->fp - (char*)s->p->code_base; + ((struct dill_pkg_1 *)pkg)->entry_offset = (short)( (char*)s->p->fp - (char*)s->p->code_base); return pkg; } -EXTERN char * +extern char * dill_finalize_package(dill_stream s, int *pkg_len) { (s->j->package_end)(s); @@ -579,7 +580,7 @@ dill_finalize_package(dill_stream s, int *pkg_len) return p; } -EXTERN int +extern int dill_code_size(dill_stream s) { char* native_base = s->p->native.code_base; @@ -589,13 +590,13 @@ dill_code_size(dill_stream s) return (int) ((char*)s->p->cur_ip - native_base); } -EXTERN void * +extern void * dill_clone_code(dill_stream s, void *new_base, int size) { return (s->j->clone_code)(s, new_base, size); } -EXTERN void * +extern void * dill_take_code(dill_stream s) { void *ret = s->p->code_base; @@ -604,7 +605,7 @@ dill_take_code(dill_stream s) return ret; } -EXTERN int +extern int dill_alloc_label(dill_stream s, char *name) { struct branch_table *t = &s->p->branch_table; @@ -619,7 +620,7 @@ dill_alloc_label(dill_stream s, char *name) return t->next_label++; } -EXTERN void dill_mark_label(dill_stream s, int label) +extern void dill_mark_label(dill_stream s, int label) { struct branch_table *t = &s->p->branch_table; int label_loc = (int) ((char*)s->p->cur_ip - (char*)s->p->code_base); @@ -634,7 +635,7 @@ EXTERN void dill_mark_label(dill_stream s, int label) } } -EXTERN int dill_is_label_mark(dill_stream s) +extern int dill_is_label_mark(dill_stream s) { struct branch_table *t = &s->p->branch_table; int i; @@ -691,7 +692,7 @@ extern void dill_mark_call_location(dill_stream s, const char *xfer_name, t->call_count++; } -EXTERN int +extern int dill_add_const(dill_stream s, void *addr, int size) { struct branch_table *t = &s->p->branch_table; @@ -767,7 +768,7 @@ translate_arg_str(const char *string, int *count) return list; } -EXTERN void +extern void dill_start_proc(dill_stream s, char *name, int ret_type, char *arg_str) { int arg_count = 0; @@ -792,7 +793,7 @@ init_code_block(dill_stream s) #ifndef MAP_ANONYMOUS #define MAP_ANONYMOUS MAP_ANON #endif - static unsigned long ps = -1; + static long ps = -1; if (ps == -1) { ps = (getpagesize ()); } @@ -908,7 +909,7 @@ reg_alloc(reg_set *regs) return reg; } -EXTERN void +extern void dill_alloc_specific(dill_stream s, dill_reg reg, int type, int class) { switch(type) { @@ -931,7 +932,7 @@ dill_alloc_specific(dill_stream s, dill_reg reg, int type, int class) } } -EXTERN void +extern void dill_dealloc_specific(dill_stream s, dill_reg reg, int type, int class) { switch(type) { @@ -954,7 +955,7 @@ dill_dealloc_specific(dill_stream s, dill_reg reg, int type, int class) } } -EXTERN void +extern void dill_raw_unavailreg(dill_stream s, int type, dill_reg reg) { if (s->p->unavail_called == 0) { @@ -973,7 +974,7 @@ dill_raw_unavailreg(dill_stream s, int type, dill_reg reg) } } -EXTERN void +extern void dill_raw_availreg(dill_stream s, int type, dill_reg reg) { switch(type) { @@ -998,7 +999,7 @@ dill_raw_availreg(dill_stream s, int type, dill_reg reg) } } -EXTERN int +extern int dill_getreg(dill_stream s, int typ) { s->p->vregs = realloc(s->p->vregs, @@ -1010,7 +1011,7 @@ dill_getreg(dill_stream s, int typ) return ((s->p->vreg_count++) + 100); } -EXTERN int +extern int dill_getvblock(dill_stream s, int size) { s->p->vregs = realloc(s->p->vregs, @@ -1022,7 +1023,7 @@ dill_getvblock(dill_stream s, int size) return ((s->p->vreg_count++) + 100); } -EXTERN int +extern int dill_raw_getreg(dill_stream s, dill_reg *reg_p, int type, int class) { int reg = -1; @@ -1048,7 +1049,6 @@ dill_raw_getreg(dill_stream s, dill_reg *reg_p, int type, int class) *reg_p = reg; return (reg != -1); } - break; default: if (class == DILL_VAR) { if ((reg = reg_alloc(&s->p->var_i)) == -1) { @@ -1072,11 +1072,10 @@ dill_raw_getreg(dill_stream s, dill_reg *reg_p, int type, int class) *reg_p = reg; return (reg != -1); } - break; } } -EXTERN void +extern void dill_raw_putreg(dill_stream s, dill_reg reg, int type) { switch (type) { @@ -1107,7 +1106,7 @@ dill_raw_putreg(dill_stream s, dill_reg reg, int type) } } -EXTERN int +extern int dill_do_reverse_vararg_push(dill_stream s) { if (s->j->do_reverse_push) { @@ -1342,7 +1341,7 @@ dill_pcompare(dill_stream s, int op_type, int data_type, dill_reg dest, dill_reg s->j->c_data[index].data2, dest, src1, src2); } -EXTERN void +extern void dill_dump_reg(dill_stream s, int typ, int reg) { s->j->print_reg(s, typ, reg); @@ -1493,7 +1492,7 @@ dump_cur_dill_insn(dill_stream s) } #endif -EXTERN void +extern void dill_dump(dill_stream s) { struct disassemble_info info; @@ -1513,7 +1512,7 @@ dill_dump(dill_stream s) int insn_count = 0; printf("\nDILL virtual instruction stream\n\n"); for (p =base; p < code_limit;) { - printf("%lx - %x - ", (unsigned long)p, (unsigned)*(int*)p); + printf("%p - %x - ", p, (unsigned)*(int*)p); l = s->p->virtual.mach_jump->print_insn(s, &info, (void *)p); printf("\n"); if (l == -1) return; @@ -1557,7 +1556,7 @@ dill_dump(dill_stream s) if (p == s->p->fp) { printf("Function entry point:\n"); } - printf("%lx - %x - ", (unsigned long)p, (unsigned)*(int*)p); + printf("%p - %x - ", p, (unsigned)*(int*)p); l = s->j->print_insn(s, &info, (void *)p); printf("\n"); if (l <= 0) return; diff --git a/thirdparty/dill/dill/scripts/ci/cmake/centos7-clang.cmake b/thirdparty/dill/dill/scripts/ci/cmake/centos7-clang.cmake new file mode 100644 index 0000000000..6f6f2e2d40 --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/centos7-clang.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} clang) +set(ENV{CXX} clang++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/cmake/centos7-gcc.cmake b/thirdparty/dill/dill/scripts/ci/cmake/centos7-gcc.cmake new file mode 100644 index 0000000000..8a1c976d8f --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/centos7-gcc.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} gcc) +set(ENV{CXX} g++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/cmake/centos7-nvhpc.cmake b/thirdparty/dill/dill/scripts/ci/cmake/centos7-nvhpc.cmake new file mode 100644 index 0000000000..30e1f6cd58 --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/centos7-nvhpc.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} nvc) +set(ENV{CXX} nvc++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/cmake/centos8-clang.cmake b/thirdparty/dill/dill/scripts/ci/cmake/centos8-clang.cmake new file mode 100644 index 0000000000..6f6f2e2d40 --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/centos8-clang.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} clang) +set(ENV{CXX} clang++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/cmake/centos8-gcc.cmake b/thirdparty/dill/dill/scripts/ci/cmake/centos8-gcc.cmake new file mode 100644 index 0000000000..8a1c976d8f --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/centos8-gcc.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} gcc) +set(ENV{CXX} g++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/cmake/centos8-intel.cmake b/thirdparty/dill/dill/scripts/ci/cmake/centos8-intel.cmake new file mode 100644 index 0000000000..b173d984ec --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/centos8-intel.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} icc) +set(ENV{CXX} icpc) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/cmake/centos8-inteloneapi.cmake b/thirdparty/dill/dill/scripts/ci/cmake/centos8-inteloneapi.cmake new file mode 100644 index 0000000000..09ca1203cd --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/centos8-inteloneapi.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} icx) +set(ENV{CXX} icpx) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/cmake/common.cmake b/thirdparty/dill/dill/scripts/ci/cmake/common.cmake new file mode 100644 index 0000000000..f0e57e6c7a --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/common.cmake @@ -0,0 +1,62 @@ +if(POLICY CMP0057) + cmake_policy(SET CMP0057 NEW) +endif() + +if(NOT CTEST_BUILD_CONFIGURATION) + set(CTEST_BUILD_CONFIGURATION Debug) +endif() + +if(NOT DEFINED NCPUS) + include(ProcessorCount) + ProcessorCount(NCPUS) +endif() +math(EXPR N2CPUS "${NCPUS}*2") +if(NOT CTEST_BUILD_FLAGS) + if(CTEST_CMAKE_GENERATOR STREQUAL "Unix Makefiles") + set(CTEST_BUILD_FLAGS "-k -j${N2CPUS}") + elseif(CTEST_CMAKE_GENERATOR STREQUAL "Ninja") + set(CTEST_BUILD_FLAGS "-k0 -j${N2CPUS}") + endif() +endif() +if(NOT PARALLEL_LEVEL IN_LIST CTEST_TEST_ARGS) + list(APPEND CTEST_TEST_ARGS PARALLEL_LEVEL ${N2CPUS}) +endif() + +if(NOT dashboard_model) + set(dashboard_model Experimental) +endif() +if(NOT dashboard_binary_name) + set(dashboard_binary_name "build") +endif() +if(NOT dashboard_track) + set(dashboard_track "Continuous Integration") +endif() +if(NOT "$ENV{CI_COMMIT_SHA}" STREQUAL "") + set(CTEST_UPDATE_VERSION_OVERRIDE "$ENV{CI_COMMIT_SHA}") + set(CTEST_UPDATE_VERSION_ONLY ON) +endif() +if(NOT "$ENV{CI_SITE_NAME}" STREQUAL "") + set(CTEST_SITE "$ENV{CI_SITE_NAME}") +endif() +if(NOT "$ENV{CI_BUILD_NAME}" STREQUAL "") + set(CTEST_BUILD_NAME "$ENV{CI_BUILD_NAME}") +endif() +if(NOT "$ENV{CI_ROOT_DIR}" STREQUAL "") + set(CTEST_DASHBOARD_ROOT "$ENV{CI_ROOT_DIR}") +endif() +if(NOT "$ENV{CI_SOURCE_DIR}" STREQUAL "") + set(CTEST_SOURCE_DIRECTORY "$ENV{CI_SOURCE_DIR}") +endif() +if(NOT "$ENV{CI_BIN_DIR}" STREQUAL "") + set(CTEST_BINARY_DIRECTORY "$ENV{CI_BIN_DIR}") +endif() + +find_program(CTEST_GIT_COMMAND git) +set(CTEST_UPDATE_COMMAND ${CTEST_GIT_COMMAND}) +set(CTEST_UPDATE_TYPE git) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/../../dashboard/dill_common.cmake) +if(ctest_build_num_warnings GREATER 0) + message(FATAL_ERROR "Found ${ctest_build_num_warnings} warnings.") +endif() diff --git a/thirdparty/dill/dill/scripts/ci/cmake/macos-clang.cmake b/thirdparty/dill/dill/scripts/ci/cmake/macos-clang.cmake new file mode 100644 index 0000000000..868b8b44b7 --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/macos-clang.cmake @@ -0,0 +1,6 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} clang) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/cmake/ubuntu1604-clang.cmake b/thirdparty/dill/dill/scripts/ci/cmake/ubuntu1604-clang.cmake new file mode 100644 index 0000000000..6f6f2e2d40 --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/ubuntu1604-clang.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} clang) +set(ENV{CXX} clang++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/cmake/ubuntu1604-gcc.cmake b/thirdparty/dill/dill/scripts/ci/cmake/ubuntu1604-gcc.cmake new file mode 100644 index 0000000000..8a1c976d8f --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/ubuntu1604-gcc.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} gcc) +set(ENV{CXX} g++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/cmake/ubuntu1804-clang.cmake b/thirdparty/dill/dill/scripts/ci/cmake/ubuntu1804-clang.cmake new file mode 100644 index 0000000000..6f6f2e2d40 --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/ubuntu1804-clang.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} clang) +set(ENV{CXX} clang++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/cmake/ubuntu1804-gcc.cmake b/thirdparty/dill/dill/scripts/ci/cmake/ubuntu1804-gcc.cmake new file mode 100644 index 0000000000..8a1c976d8f --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/ubuntu1804-gcc.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} gcc) +set(ENV{CXX} g++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/cmake/ubuntu1804-intel.cmake b/thirdparty/dill/dill/scripts/ci/cmake/ubuntu1804-intel.cmake new file mode 100644 index 0000000000..b173d984ec --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/ubuntu1804-intel.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} icc) +set(ENV{CXX} icpc) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/cmake/ubuntu1804-inteloneapi.cmake b/thirdparty/dill/dill/scripts/ci/cmake/ubuntu1804-inteloneapi.cmake new file mode 100644 index 0000000000..09ca1203cd --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/ubuntu1804-inteloneapi.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} icx) +set(ENV{CXX} icpx) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/cmake/ubuntu2004-clang.cmake b/thirdparty/dill/dill/scripts/ci/cmake/ubuntu2004-clang.cmake new file mode 100644 index 0000000000..6f6f2e2d40 --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/ubuntu2004-clang.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} clang) +set(ENV{CXX} clang++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/cmake/ubuntu2004-gcc.cmake b/thirdparty/dill/dill/scripts/ci/cmake/ubuntu2004-gcc.cmake new file mode 100644 index 0000000000..8a1c976d8f --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/ubuntu2004-gcc.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} gcc) +set(ENV{CXX} g++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/cmake/ubuntu2004-nvhpc.cmake b/thirdparty/dill/dill/scripts/ci/cmake/ubuntu2004-nvhpc.cmake new file mode 100644 index 0000000000..30e1f6cd58 --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/ubuntu2004-nvhpc.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(ENV{CC} nvc) +set(ENV{CXX} nvc++) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/unix-common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/cmake/unix-common.cmake b/thirdparty/dill/dill/scripts/ci/cmake/unix-common.cmake new file mode 100644 index 0000000000..93631c10cd --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/unix-common.cmake @@ -0,0 +1,11 @@ +# Client maintainer: chuck.atkins@kitware.com + +string(APPEND dashboard_cache " +") + +if(NOT CTEST_CMAKE_GENERATOR) + set(CTEST_CMAKE_GENERATOR "Unix Makefiles") +endif() + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/cmake/windows-common.cmake b/thirdparty/dill/dill/scripts/ci/cmake/windows-common.cmake new file mode 100644 index 0000000000..1b227e5395 --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/windows-common.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +string(APPEND dashboard_cache " +") + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/cmake/windows-vs2019-clang.cmake b/thirdparty/dill/dill/scripts/ci/cmake/windows-vs2019-clang.cmake new file mode 100644 index 0000000000..3667faf3c1 --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/windows-vs2019-clang.cmake @@ -0,0 +1,8 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(CTEST_CMAKE_GENERATOR "Visual Studio 16") +set(CTEST_CMAKE_GENERATOR_PLATFORM x64) +set(CTEST_CMAKE_GENERATOR_TOOLSET ClangCL) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/windows-common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/cmake/windows-vs2019-msvc.cmake b/thirdparty/dill/dill/scripts/ci/cmake/windows-vs2019-msvc.cmake new file mode 100644 index 0000000000..c21aff334b --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/cmake/windows-vs2019-msvc.cmake @@ -0,0 +1,7 @@ +# Client maintainer: chuck.atkins@kitware.com + +set(CTEST_CMAKE_GENERATOR "Visual Studio 16") +set(CTEST_CMAKE_GENERATOR_PLATFORM x64) + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/windows-common.cmake) diff --git a/thirdparty/dill/dill/scripts/ci/gh-actions/run.sh b/thirdparty/dill/dill/scripts/ci/gh-actions/run.sh new file mode 100755 index 0000000000..fdc910dddd --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/gh-actions/run.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +readlinkf() { perl -MCwd -e 'print Cwd::abs_path shift' "$1"; } + +set -e + +STEP=$1 +shift + +export CI_SITE_NAME="GitHub Actions" +if [ "${GITHUB_EVENT_NAME}" = "pull_request" ] +then + GH_PR_NUMBER=$(expr "${GITHUB_REF}" : 'refs/pull/\([^/]*\)') + export CI_BUILD_NAME="pr${GH_PR_NUMBER}_${GITHUB_HEAD_REF}_${GH_YML_JOBNAME}_${GH_YML_BUILDTYPE}" +else + export CI_BUILD_NAME="${GITHUB_REF#refs/heads/}_${GH_YML_JOBNAME}_${GH_YML_BUILDTYPE}" +fi +if [[ "${RUNNER_OS}" =~ "Windows" ]] +then + export CI_ROOT_DIR="${GITHUB_WORKSPACE//\\//}" + export CI_SOURCE_DIR="${GITHUB_WORKSPACE//\\//}/source" +else + export CI_ROOT_DIR="${GITHUB_WORKSPACE}" + export CI_SOURCE_DIR="${GITHUB_WORKSPACE}/source" +fi +export CI_BIN_DIR="${CI_ROOT_DIR}/build" + +export CI_COMMIT_SHA=${GH_YML_SHA} + +if command -v ctest3 >/dev/null +then + CTEST=ctest3 +else + CTEST=ctest +fi + +# Update and Test steps enable an extra step +CTEST_STEP_ARGS="" +case ${STEP} in + test) CTEST_STEP_ARGS="${CTEST_STEP_ARGS} -Ddashboard_do_end=ON" ;; +esac +CTEST_STEP_ARGS="${CTEST_STEP_ARGS} -Ddashboard_do_${STEP}=ON" + +echo "********** Environment **********" + +env | sort + +echo "********** Running CTest **********" + +${CTEST} -VV \ + -S ${CI_SOURCE_DIR}/scripts/ci/cmake/${GH_YML_JOBNAME}.cmake \ + -DCTEST_BUILD_CONFIGURATION=${GH_YML_BUILDTYPE} \ + -Ddashboard_do_submit=ON \ + -Ddashboard_full=OFF \ + ${CTEST_STEP_ARGS} \ + "$@" + diff --git a/thirdparty/dill/dill/scripts/ci/scripts/post-cdash-status.sh b/thirdparty/dill/dill/scripts/ci/scripts/post-cdash-status.sh new file mode 100755 index 0000000000..766ea9a90b --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/scripts/post-cdash-status.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +if [ $# -eq 0 ] +then + echo "$0 org/repo [SHA [TOKEN]]" + exit 1 +fi + +GH_REPO=$1 +if [ $# -gt 1 ] +then + GH_SHA=$2 + if [ $# -gt 2 ] + then + GH_TOKEN=$3 + if [ $# -gt 3 ] + then + echo "$0 org/repo [SHA [TOKEN]]" + exit 1 + fi + fi +fi + +if [ -z "${GH_SHA}" ] +then + echo "Error: GH_SHA is undefined" + exit 2 +fi +if [ -z "${GH_TOKEN}" ] +then + echo "Error: GH_TOKEN is undefined" + exit 3 +fi + +cdash_url() { + printf 'https://open.cdash.org/index.php?project=%s&subproject=%s&filtercount=1&showfilters=1&field1=revision&compare1=61&value1=%s' $1 $2 $3 +} + +build_status_body() { +cat < /etc/yum.repos.d/group_git-maint-git-epel-7.repo + ;; + centos8*) + curl -L https://copr.fedorainfracloud.org/coprs/g/git-maint/git/repo/epel-8/group_git-maint-git-epel-8.repo > /etc/yum.repos.d/group_git-maint-git-epel-8.repo + ;; + ubuntu*) + export DEBIAN_FRONTEND=noninteractive + add-apt-repository ppa:git-core/ppa -y + apt-get update + ;; +esac +${PKG_CMD} install -y git + +######################################## +# Compilers +######################################## +case ${GH_YML_JOBNAME} in + centos*-clang) PKGS="clang gcc gcc-c++" ;; + centos*-gcc) PKGS="gcc gcc-c++" ;; + centos*-nvhpc) PKGS="gcc gcc-c++" ;; + ubuntu*-clang) PKGS="clang gcc g++" ;; + ubuntu*-gcc) PKGS="gcc g++" ;; + ubuntu*-nvhpc) PKGS="gcc g++" ;; +esac +${PKG_CMD} install -y ${PKGS} + + +case ${GH_YML_JOBNAME} in + *-clang) export CC=clang CXX=clang++ ;; + *-gcc) export CC=gcc CXX=g++ ;; + *-nvhpc) export CC=nvc CXX=nvc++ ;; +esac + +######################################## +# CMake +######################################## +FILENAME=$(curl https://cmake.org/files/LatestRelease/cmake-latest-files-v1.json 2>/dev/null | grep 'cmake.*sh' | sed -n 's|.*"\(cmake.*x86_64.sh\).*|\1|p') +VERSION=$(echo ${FILENAME} | sed 's|cmake-\([^\-]*\).*|\1|') +curl -L https://github.com/Kitware/CMake/releases/download/v${VERSION}/${FILENAME} > cmake.sh +chmod +x cmake.sh +./cmake.sh --skip-license --exclude-subdir --prefix=/usr/local +rm -f cmake.sh diff --git a/thirdparty/dill/dill/scripts/ci/setup/macos.sh b/thirdparty/dill/dill/scripts/ci/setup/macos.sh new file mode 100755 index 0000000000..a9bf588e2f --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/setup/macos.sh @@ -0,0 +1 @@ +#!/bin/bash diff --git a/thirdparty/dill/dill/scripts/ci/setup/windows.sh b/thirdparty/dill/dill/scripts/ci/setup/windows.sh new file mode 100755 index 0000000000..a9bf588e2f --- /dev/null +++ b/thirdparty/dill/dill/scripts/ci/setup/windows.sh @@ -0,0 +1 @@ +#!/bin/bash diff --git a/thirdparty/dill/dill/scripts/dashboard/common.cmake b/thirdparty/dill/dill/scripts/dashboard/common.cmake new file mode 100644 index 0000000000..4b50bf291e --- /dev/null +++ b/thirdparty/dill/dill/scripts/dashboard/common.cmake @@ -0,0 +1,515 @@ +#------------------------------------------------------------------------------# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +#------------------------------------------------------------------------------# +# +# "Universal" Dashboard Script +# +# This script contains basic dashboard driver code common to all +# clients and projects. It is a combination of the universal.cmake script in +# the Kitware DashboardScriptsNG repo and cmake_common.cmake used by CMake +# dashboards. +# +# Create a project-specific common script with code of the following form, +# where the final line includes this script. +# +# set(CTEST_PROJECT_NAME "OpenChemistry") +# set(CTEST_DROP_SITE "cdash.openchemistry.org") +# +# set(dashboard_git_url "git://source.openchemistry.org/openchemistry.git") +# set(dashboard_root_name "MyTests") +# set(dashboard_source_name "openchemistry") +# +# get_filename_component(dir ${CMAKE_CURRENT_LIST_FILE} PATH) +# include(${dir}/universal.cmake) +# +# The following variables may be set before including this script +# to configure it: +# +# dashboard_model = Nightly | Experimental +# dashboard_root_name = Change name of "My Tests" directory +# dashboard_source_name = Name of source directory (CMake) +# dashboard_binary_name = Name of binary directory (CMake-build) +# dashboard_cache = Initial CMakeCache.txt file content +# dashboard_track = The name of the CDash "Track" to submit to + +# dashboard_do_checkout = True to enable source checkout via git +# dashboard_do_update = True to enable the Update step +# dashboard_do_configure = True to enable the Configure step +# dashboard_do_build = True to enable the Build step +# dashboard_do_test = True to enable the Test step +# dashboard_do_coverage = True to enable coverage (ex: gcov) +# dashboard_do_memcheck = True to enable memcheck (ex: valgrind) +# dashboard_do_submit = Submit each step (ON) +# dashboard_do_submit_only = Only submit step results, do nto run them (OFF) + +# CTEST_GIT_COMMAND = path to git command-line client +# CTEST_BUILD_FLAGS = build tool arguments (ex: -j2) +# CTEST_DASHBOARD_ROOT = Where to put source and build trees +# CTEST_TEST_CTEST = Whether to run long CTestTest* tests +# CTEST_TEST_TIMEOUT = Per-test timeout length +# CTEST_TEST_ARGS = ctest_test args (ex: PARALLEL_LEVEL 4) +# CMAKE_MAKE_PROGRAM = Path to "make" tool to use +# +# Options to configure Git: +# dashboard_git_url = Custom git clone url +# dashboard_git_branch = Custom remote branch to track +# dashboard_git_crlf = Value of core.autocrlf for repository +# +# For Makefile generators the script may be executed from an +# environment already configured to use the desired compilers. +# Alternatively the environment may be set at the top of the script: +# +# set(ENV{CC} /path/to/cc) # C compiler +# set(ENV{CXX} /path/to/cxx) # C++ compiler +# set(ENV{FC} /path/to/fc) # Fortran compiler (optional) +# set(ENV{LD_LIBRARY_PATH} /path/to/vendor/lib) # (if necessary) + +cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR) + +if(NOT DEFINED dashboard_full) + set(dashboard_full TRUE) +endif() + +# Initialize all build steps to "ON" +if(NOT DEFINED dashboard_do_update) + set(dashboard_do_update ${dashboard_full}) +endif() + +if(NOT DEFINED dashboard_do_checkout) + set(dashboard_do_checkout ${dashboard_full}) +endif() + +if(NOT DEFINED dashboard_do_configure) + set(dashboard_do_configure ${dashboard_full}) +endif() + +if(NOT DEFINED dashboard_do_build) + set(dashboard_do_build ${dashboard_full}) +endif() + +if(NOT DEFINED dashboard_do_test) + set(dashboard_do_test ${dashboard_full}) +endif() + +# Default code coverage and memtesting to off +if(NOT DEFINED dashboard_do_coverage) + set(dashboard_do_coverage FALSE) +endif() + +if(NOT DEFINED dashboard_do_memcheck) + set(dashboard_do_memcheck FALSE) +endif() + +if(NOT DEFINED dashboard_fresh) + if(dashboard_full OR dashboard_do_update) + set(dashboard_fresh TRUE) + else() + set(dashboard_fresh FALSE) + endif() +endif() + +if(NOT DEFINED dashboard_do_submit_only) + set(dashboard_do_submit_only FALSE) +endif() + +if(NOT DEFINED dashboard_do_submit) + set(dashboard_do_submit TRUE) +endif() + +if(NOT DEFINED CTEST_PROJECT_NAME) + message(FATAL_ERROR "project-specific script including 'universal.cmake' should set CTEST_PROJECT_NAME") +endif() + +if(NOT DEFINED dashboard_user_home) + set(dashboard_user_home "$ENV{HOME}") +endif() + +# Select the top dashboard directory. +if(NOT DEFINED dashboard_root_name) + set(dashboard_root_name "My Tests") +endif() +if(NOT DEFINED CTEST_DASHBOARD_ROOT) + get_filename_component(CTEST_DASHBOARD_ROOT "${CTEST_SCRIPT_DIRECTORY}/../${dashboard_root_name}" ABSOLUTE) +endif() + +# Select the model (Nightly, Experimental, Continuous). +if(NOT DEFINED dashboard_model) + set(dashboard_model Nightly) +endif() +if(NOT "${dashboard_model}" MATCHES "^(Nightly|Experimental)$") + message(FATAL_ERROR "dashboard_model must be Nightly or Experimental") +endif() + +# Default to a Debug build. +if(NOT DEFINED CTEST_BUILD_CONFIGURATION) + set(CTEST_BUILD_CONFIGURATION Debug) +endif() + +if(NOT DEFINED CTEST_CONFIGURATION_TYPE) + set(CTEST_CONFIGURATION_TYPE ${CTEST_BUILD_CONFIGURATION}) +endif() + +# Choose CTest reporting mode. +if(NOT "${CTEST_CMAKE_GENERATOR}" MATCHES "Make|Ninja") + # Launchers work only with Makefile and Ninja generators. + set(CTEST_USE_LAUNCHERS 0) +elseif(NOT DEFINED CTEST_USE_LAUNCHERS) + # The setting is ignored by CTest < 2.8 so we need no version test. + set(CTEST_USE_LAUNCHERS 1) +endif() + +# Configure testing. +if(NOT DEFINED CTEST_TEST_CTEST) + set(CTEST_TEST_CTEST 1) +endif() +if(NOT CTEST_TEST_TIMEOUT) + set(CTEST_TEST_TIMEOUT 1500) +endif() + +# Select Git source to use. +if(dashboard_do_checkout) + if(NOT DEFINED dashboard_git_url) + message(FATAL_ERROR "project-specific script including 'universal.cmake' should set dashboard_git_url") + endif() + if(NOT DEFINED dashboard_git_branch) + set(dashboard_git_branch master) + endif() + if(NOT DEFINED dashboard_git_crlf) + if(UNIX) + set(dashboard_git_crlf false) + else() + set(dashboard_git_crlf true) + endif() + endif() + + # Look for a GIT command-line client. + if(NOT DEFINED CTEST_GIT_COMMAND) + find_program(CTEST_GIT_COMMAND + NAMES git git.cmd + PATH_SUFFIXES Git/cmd Git/bin + ) + endif() + if(NOT CTEST_GIT_COMMAND) + message(FATAL_ERROR "CTEST_GIT_COMMAND not available!") + endif() +endif() + +# Select a source directory name. +if(NOT DEFINED CTEST_SOURCE_DIRECTORY) + if(DEFINED dashboard_source_name) + set(CTEST_SOURCE_DIRECTORY ${CTEST_DASHBOARD_ROOT}/${dashboard_source_name}) + else() + set(CTEST_SOURCE_DIRECTORY ${CTEST_DASHBOARD_ROOT}/${CTEST_PROJECT_NAME}) + endif() +endif() + +# Select a build directory name. +if(NOT DEFINED CTEST_BINARY_DIRECTORY) + if(DEFINED dashboard_binary_name) + set(CTEST_BINARY_DIRECTORY ${CTEST_DASHBOARD_ROOT}/${dashboard_binary_name}) + else() + set(CTEST_BINARY_DIRECTORY ${CTEST_SOURCE_DIRECTORY}-build) + endif() +endif() + +macro(dashboard_git) + execute_process( + COMMAND ${CTEST_GIT_COMMAND} ${ARGN} + WORKING_DIRECTORY "${CTEST_SOURCE_DIRECTORY}" + OUTPUT_VARIABLE dashboard_git_output + ERROR_VARIABLE dashboard_git_output + RESULT_VARIABLE dashboard_git_failed + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_STRIP_TRAILING_WHITESPACE + ) +endmacro() + +if(dashboard_do_checkout) + # Delete source tree if it is incompatible with current VCS. + if(EXISTS ${CTEST_SOURCE_DIRECTORY}) + if(NOT EXISTS "${CTEST_SOURCE_DIRECTORY}/.git") + set(vcs_refresh "because it is not managed by git.") + else() + execute_process( + COMMAND ${CTEST_GIT_COMMAND} reset --hard + WORKING_DIRECTORY "${CTEST_SOURCE_DIRECTORY}" + OUTPUT_VARIABLE output + ERROR_VARIABLE output + RESULT_VARIABLE failed + ) + if(failed) + set(vcs_refresh "because its .git may be corrupted.") + endif() + endif() + if(vcs_refresh AND "${CTEST_SOURCE_DIRECTORY}" MATCHES "/CMake[^/]*") + message("Deleting source tree\n") + message(" ${CTEST_SOURCE_DIRECTORY}\n${vcs_refresh}") + file(REMOVE_RECURSE "${CTEST_SOURCE_DIRECTORY}") + endif() + endif() + + # Support initial checkout if necessary. + if(NOT EXISTS "${CTEST_SOURCE_DIRECTORY}" + AND NOT DEFINED CTEST_CHECKOUT_COMMAND) + # Generate an initial checkout script. + get_filename_component(_name "${CTEST_SOURCE_DIRECTORY}" NAME) + set(ctest_checkout_script ${CTEST_DASHBOARD_ROOT}/${_name}-init.cmake) + file(WRITE ${ctest_checkout_script} "# git repo init script for ${_name} +execute_process( + COMMAND \"${CTEST_GIT_COMMAND}\" clone -n -- \"${dashboard_git_url}\" + \"${CTEST_SOURCE_DIRECTORY}\" + ) +if(EXISTS \"${CTEST_SOURCE_DIRECTORY}/.git\") + execute_process( + COMMAND \"${CTEST_GIT_COMMAND}\" config core.autocrlf ${dashboard_git_crlf} + WORKING_DIRECTORY \"${CTEST_SOURCE_DIRECTORY}\" + ) + execute_process( + COMMAND \"${CTEST_GIT_COMMAND}\" fetch + WORKING_DIRECTORY \"${CTEST_SOURCE_DIRECTORY}\" + ) + execute_process( + COMMAND \"${CTEST_GIT_COMMAND}\" checkout ${dashboard_git_branch} + WORKING_DIRECTORY \"${CTEST_SOURCE_DIRECTORY}\" + ) +endif()" + ) + set(CTEST_CHECKOUT_COMMAND "\"${CMAKE_COMMAND}\" -P \"${ctest_checkout_script}\"") + elseif(EXISTS "${CTEST_SOURCE_DIRECTORY}/.git") + # Upstream URL. + dashboard_git(config --get remote.origin.url) + if(NOT dashboard_git_output STREQUAL "${dashboard_git_url}") + dashboard_git(config remote.origin.url "${dashboard_git_url}") + endif() + + # Local checkout. + dashboard_git(symbolic-ref HEAD) + if(NOT dashboard_git_output STREQUAL "${dashboard_git_branch}") + dashboard_git(checkout ${dashboard_git_branch}) + if(dashboard_git_failed) + message(FATAL_ERROR "Failed to checkout branch ${dashboard_git_branch}:\n${dashboard_git_output}") + endif() + endif() + endif() +endif() + +#----------------------------------------------------------------------------- + +# Check for required variables. +foreach(req + CTEST_CMAKE_GENERATOR + CTEST_SITE + CTEST_BUILD_NAME + ) + if(NOT DEFINED ${req}) + message(FATAL_ERROR "The containing script must set ${req}") + endif() +endforeach(req) + +# Print summary information. +set(vars "") +foreach(v + CTEST_SITE + CTEST_BUILD_NAME + CTEST_SOURCE_DIRECTORY + CTEST_BINARY_DIRECTORY + CTEST_CMAKE_GENERATOR + CTEST_BUILD_CONFIGURATION + CTEST_GIT_COMMAND + CTEST_CHECKOUT_COMMAND + CTEST_CONFIGURE_COMMAND + CTEST_SCRIPT_DIRECTORY + CTEST_USE_LAUNCHERS + ) + set(vars "${vars} ${v}=[${${v}}]\n") +endforeach(v) +message("Dashboard script configuration:\n${vars}\n") + +# Avoid non-ascii characters in tool output. +set(ENV{LC_ALL} C) + +# Helper macro to write the initial cache. +macro(write_cache) + set(cache_build_type "") + set(cache_make_program "") + if(CTEST_CMAKE_GENERATOR MATCHES "Make|Ninja") + set(cache_build_type CMAKE_BUILD_TYPE:STRING=${CTEST_BUILD_CONFIGURATION}) + if(CMAKE_MAKE_PROGRAM) + set(cache_make_program CMAKE_MAKE_PROGRAM:FILEPATH=${CMAKE_MAKE_PROGRAM}) + endif() + endif() + file(WRITE ${CTEST_BINARY_DIRECTORY}/CMakeCache.txt " +SITE:STRING=${CTEST_SITE} +BUILDNAME:STRING=${CTEST_BUILD_NAME} +CTEST_TEST_CTEST:BOOL=${CTEST_TEST_CTEST} +CTEST_USE_LAUNCHERS:BOOL=${CTEST_USE_LAUNCHERS} +DART_TESTING_TIMEOUT:STRING=${CTEST_TEST_TIMEOUT} +GIT_EXECUTABLE:FILEPATH=${CTEST_GIT_COMMAND} +${cache_build_type} +${cache_make_program} +${dashboard_cache} +") +endmacro(write_cache) + +if(COMMAND dashboard_hook_init) + dashboard_hook_init() +endif() + +if(dashboard_fresh) + if(EXISTS "${CTEST_BINARY_DIRECTORY}") + message("Clearing build tree...") + ctest_empty_binary_directory("${CTEST_BINARY_DIRECTORY}") + else() + file(MAKE_DIRECTORY "${CTEST_BINARY_DIRECTORY}") + endif() + message("Starting fresh build...") + write_cache() +endif() + +# Start a new submission. +if(dashboard_track) + set(dashboard_track_arg TRACK "${dashboard_track}") +endif() +if(dashboard_fresh) + if(COMMAND dashboard_hook_start) + dashboard_hook_start() + endif() + message("Calling ctest_start") + ctest_start(${dashboard_model} ${dashboard_track_arg}) + if(dashboard_do_submit) + message("Calling ctest_submit") + ctest_submit(PARTS Start) + endif() + if(COMMAND dashboard_hook_started) + dashboard_hook_started() + endif() +else() + message("Calling ctest_start(APPEND)") + ctest_start(${dashboard_model} ${dashboard_track_arg} APPEND) +endif() + +# Look for updates. +if(dashboard_do_update) + if(NOT dashboard_do_submit_only) + if(COMMAND dashboard_hook_update) + dashboard_hook_update() + endif() + message("Calling ctest_update") + ctest_update(RETURN_VALUE count) + set(CTEST_CHECKOUT_COMMAND) # checkout on first iteration only + message("Found ${count} changed files") + endif() + + if(dashboard_do_submit) + if(CTEST_SUBMIT_NOTES) + message("Submitting dashboard scripts as Notes") + # Send the main script as a note while submitting the Update part + set(CTEST_NOTES_FILES + ${CTEST_UPDATE_NOTES_FILES} + "${CMAKE_CURRENT_LIST_FILE}") + message("Calling ctest_submit") + ctest_submit(PARTS Update Notes) + unset(CTEST_NOTES_FILES) + else() + message("Skipping notes submission for Update step") + message("Calling ctest_submit") + ctest_submit(PARTS Update) + endif() + endif() +endif() + +if(dashboard_do_configure) + if(NOT dashboard_do_submit_only) + if(COMMAND dashboard_hook_configure) + dashboard_hook_configure() + endif() + message("Calling ctest_configure") + ctest_configure(${dashboard_configure_args}) + endif() + if(dashboard_do_submit) + if(CTEST_SUBMIT_NOTES) + message("Submitting CMakeCache.txt as Notes") + set(CTEST_NOTES_FILES "${CTEST_BINARY_DIRECTORY}/CMakeCache.txt") + message("Calling ctest_submit") + ctest_submit(PARTS Configure Notes) + unset(CTEST_NOTES_FILES) + else() + message("Skipping notes submission for Configure step") + message("Calling ctest_submit") + ctest_submit(PARTS Configure) + endif() + endif() +endif() + +ctest_read_custom_files(${CTEST_BINARY_DIRECTORY}) + +if(dashboard_do_build) + if(NOT dashboard_do_submit_only) + if(COMMAND dashboard_hook_build) + dashboard_hook_build() + endif() + message("Calling ctest_build") + ctest_build( + NUMBER_WARNINGS ctest_build_num_warnings + ) + endif() + if(dashboard_do_submit) + message("Calling ctest_submit") + ctest_submit(PARTS Build) + endif() +endif() + +if(dashboard_do_test) + if(NOT dashboard_do_submit_only) + if(COMMAND dashboard_hook_test) + dashboard_hook_test() + endif() + message("Calling ctest_test") + ctest_test(${CTEST_TEST_ARGS} RETURN_VALUE TEST_RESULTS) + if(${TEST_RESULTS} EQUAL 0) + message("ctest test results return value: ${TEST_RESULTS}") + else() + message(SEND_ERROR "Some tests failed") + endif() + endif() + if(dashboard_do_submit) + message("Calling ctest_submit") + ctest_submit(PARTS Test) + endif() +endif() + +if(dashboard_do_coverage) + if(NOT dashboard_do_submit_only) + if(COMMAND dashboard_hook_coverage) + dashboard_hook_coverage() + endif() + message("Calling ctest_coverage") + ctest_coverage() + endif() + if(dashboard_do_submit) + message("Calling ctest_submit") + ctest_submit(PARTS Coverage) + endif() +endif() + +if(dashboard_do_memcheck) + if(NOT dashboard_do_submit_only) + if(COMMAND dashboard_hook_memcheck) + dashboard_hook_memcheck() + endif() + message("Calling ctest_memcheck") + ctest_memcheck(RETURN_VALUE MEMCHECK_RETURN DEFECT_COUNT MEMCHECK_DEFECTS) + if(NOT (MEMCHECK_DEFECTS EQUAL 0)) + message(SEND_ERROR "ctest memcheck defects found: ${MEMCHECK_DEFECTS}") + endif() + endif() + if(dashboard_do_submit) + message("Calling ctest_submit") + ctest_submit(PARTS MemCheck) + endif() +endif() + +if(COMMAND dashboard_hook_end) + dashboard_hook_end() +endif() diff --git a/thirdparty/dill/dill/scripts/dashboard/dill_common.cmake b/thirdparty/dill/dill/scripts/dashboard/dill_common.cmake new file mode 100644 index 0000000000..1d6294b9f2 --- /dev/null +++ b/thirdparty/dill/dill/scripts/dashboard/dill_common.cmake @@ -0,0 +1,111 @@ +#------------------------------------------------------------------------------# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +#------------------------------------------------------------------------------# +# +# DILL Common Dashboard Script +# +# This script contains basic dashboard driver code common to all +# clients. +# +# # Client maintainer: me@mydomain.net +# set(CTEST_SITE "machine.site") +# set(CTEST_BUILD_NAME "Platform-Compiler") +# set(CTEST_CONFIGURATION_TYPE Debug) +# set(CTEST_CMAKE_GENERATOR "Unix Makefiles") +# include(${CTEST_SCRIPT_DIRECTORY}/dill_common.cmake) +# +# Then run a scheduled task (cron job) with a command line such as +# +# ctest -S ~/Dashboards/Scripts/my_dashboard.cmake -V +# +# By default the source and build trees will be placed in the path +# "../My Tests/" relative to your script location. +# +# The following variables may be set before including this script +# to configure it: +# +# dashboard_model = Nightly | Experimental +# dashboard_root_name = Change name of "MyTests" directory +# dashboard_source_name = Name of source directory (dill) +# dashboard_binary_name = Name of binary directory (dill-build) +# dashboard_cache = Initial CMakeCache.txt file content + +# dashboard_do_checkout = True to enable source checkout via git +# dashboard_do_update = True to enable source update +# dashboard_do_configure = True to enable the Configure step +# dashboard_do_build = True to enable the Build step +# dashboard_do_test = True to enable the Test step +# dashboard_do_coverage = True to enable coverage (ex: gcov) +# dashboard_do_memcheck = True to enable memcheck (ex: valgrind) + +# CTEST_GIT_COMMAND = path to git command-line client +# CTEST_BUILD_FLAGS = build tool arguments (ex: -j2) +# CTEST_DASHBOARD_ROOT = Where to put source and build trees +# CTEST_TEST_CTEST = Whether to run long CTestTest* tests +# CTEST_TEST_TIMEOUT = Per-test timeout length +# CTEST_TEST_ARGS = ctest_test args (ex: PARALLEL_LEVEL 4) +# CMAKE_MAKE_PROGRAM = Path to "make" tool to use +# +# Options to configure Git: +# dashboard_git_url = Custom git clone url +# dashboard_git_branch = Custom remote branch to track +# dashboard_git_crlf = Value of core.autocrlf for repository +# + +# For Makefile generators the script may be executed from an +# environment already configured to use the desired compilers. +# Alternatively the environment may be set at the top of the script: +# +# set(ENV{CC} /path/to/cc) # C compiler +# set(ENV{CXX} /path/to/cxx) # C++ compiler +# set(ENV{FC} /path/to/fc) # Fortran compiler (optional) +# set(ENV{LD_LIBRARY_PATH} /path/to/vendor/lib) # (if necessary) + +set(CTEST_PROJECT_NAME "DILL") +set(CTEST_DROP_SITE "open.cdash.org") +if(NOT dashboard_git_url) + set(dashboard_git_url "https://github.com/GTkorvo/dill.git") +endif() + +if(NOT DEFINED CTEST_TEST_TIMEOUT) + set(CTEST_TEST_TIMEOUT 120) +endif() +if(NOT DEFINED CTEST_SUBMIT_NOTES) + set(CTEST_SUBMIT_NOTES TRUE) +endif() +if(NOT DEFINED dashboard_source_name) + set(dashboard_source_name "dill") +endif() +if(NOT DEFINED dashboard_model) + set(dashboard_model Experimental) +endif() +if(NOT DEFINED dashboard_do_submit) + set(dashboard_do_submit FALSE) +endif() + +# Setup defaults for various CTEST settings +if(NOT DEFINED CTEST_SITE) + cmake_host_system_information(RESULT CTEST_SITE QUERY FQDN) +endif() +if(NOT DEFINED CTEST_BUILD_NAME) + set(CTEST_BUILD_NAME "generic-build") +endif() +if(NOT DEFINED CTEST_SOURCE_DIRECTORY) + set(CTEST_SOURCE_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/../..) +endif() +if(NOT DEFINED CTEST_BINARY_DIRECTORY) + set(CTEST_BINARY_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${CTEST_BUILD_NAME}") +endif() +if(NOT DEFINED CTEST_CMAKE_GENERATOR) + set(CTEST_CMAKE_GENERATOR "Unix Makefiles") +endif() + +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/common.cmake) + +if(CTEST_BUILD_WARNINGS_AS_ERRORS) + if(ctest_build_num_warnings GREATER 0) + message(FATAL_ERROR "Found ${ctest_build_num_warnings} build warnings.") + endif() +endif() diff --git a/thirdparty/dill/dill/tests/CMakeLists.txt b/thirdparty/dill/dill/tests/CMakeLists.txt index 32678ff4ed..8662e65a31 100644 --- a/thirdparty/dill/dill/tests/CMakeLists.txt +++ b/thirdparty/dill/dill/tests/CMakeLists.txt @@ -1,4 +1,3 @@ -cmake_minimum_required(VERSION 2.8) set(TESTS regress ctest call-test t1 pkg_test) @@ -11,7 +10,7 @@ include_directories(BEFORE add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/regress.c - COMMAND perl -w + COMMAND ${PERL_EXECUTABLE} -w ${CMAKE_CURRENT_SOURCE_DIR}/test-gen ${TEST_PERL_FLAGS} > regress.c MAIN_DEPENDENCY ${CMAKE_CURRENT_SOURCE_DIR}/test-gen @@ -19,7 +18,7 @@ add_custom_command( add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/call-test.c - COMMAND perl -w + COMMAND ${PERL_EXECUTABLE} -w ${CMAKE_CURRENT_SOURCE_DIR}/call-gen ${TEST_PERL_FLAGS} > call-test.c MAIN_DEPENDENCY ${CMAKE_CURRENT_SOURCE_DIR}/call-gen ) @@ -29,8 +28,11 @@ foreach(TEST ${TESTS}) add_test(NAME dill_${TEST} COMMAND dill_${TEST} ${TEST_PERL_FLAGS}) endforeach() -target_link_libraries(dill_regress m) -set_target_properties(dill_regress PROPERTIES COMPILE_FLAGS "-O0") +if(NOT (CMAKE_C_COMPILER_ID MATCHES "MSVC" OR + CMAKE_C_SIMULATE_ID MATCHES "MSVC")) + target_link_libraries(dill_regress m) + set_target_properties(dill_regress PROPERTIES COMPILE_FLAGS "-O0") +endif() add_executable(cplus cplus.cc) target_link_libraries(cplus dill ${DIS_LIBS}) diff --git a/thirdparty/dill/dill/tests/call-gen b/thirdparty/dill/dill/tests/call-gen index 89b0df7029..0c8d5da4ed 100644 --- a/thirdparty/dill/dill/tests/call-gen +++ b/thirdparty/dill/dill/tests/call-gen @@ -168,21 +168,15 @@ int main(int argc, char *argv[]) { long s1l, s2l; float s1f, s2f; double s1d, s2d; - char *s1p, *s2p; - int i, max_iters = 10; + int i; int verbose = 0; for (i=1; i < argc; i++) { if (strcmp(argv[i], "-v") == 0) { verbose++; - } else { - max_iters = atoi(argv[i]); } } - s1p = (void *)(long)rand(); - s2p = (void *)(long)rand(); - s1i = rand() - rand(); s2i = rand() - rand(); if(!(s2i = rand() - rand())) diff --git a/thirdparty/dill/dill/tests/stest.c b/thirdparty/dill/dill/tests/stest.c index e4697d34e2..db89d57c2b 100644 --- a/thirdparty/dill/dill/tests/stest.c +++ b/thirdparty/dill/dill/tests/stest.c @@ -6,9 +6,11 @@ #include #endif #ifdef STDC_HEADERS -#include #include #endif +#ifdef HAVE_UNISTD_H +#include +#endif #include "../config.h" #include "dill.h" @@ -120,7 +122,7 @@ int main() { #endif { int size = dill_code_size(s); - static unsigned long ps = -1; + static long ps = -1; if (ps == -1) { ps = (getpagesize ()); } diff --git a/thirdparty/dill/dill/tests/t1.c b/thirdparty/dill/dill/tests/t1.c index 61cb83c24c..ee57b7a3bd 100644 --- a/thirdparty/dill/dill/tests/t1.c +++ b/thirdparty/dill/dill/tests/t1.c @@ -19,6 +19,7 @@ int main(int argc, char **argv) h = dill_finalize(s); func = (int (*)())dill_get_fp(h); if (verbose) dill_dump(s); + fprintf(stderr, "T1, func is %p\n", (void*) func); result = func(); dill_free_handle(h); diff --git a/thirdparty/dill/dill/tests/test-gen b/thirdparty/dill/dill/tests/test-gen index e95f140f19..22c2a134de 100644 --- a/thirdparty/dill/dill/tests/test-gen +++ b/thirdparty/dill/dill/tests/test-gen @@ -1,10 +1,12 @@ $FT = "f"; $DT = "d"; +$MSVC_LONG = 0; while ($_ = $ARGV[0]) { shift; last if /^--$/; if (/^-no_float/) {$FT=""; $DT="";} + if (/^-msvc_long/) {$MSVC_LONG = 1;} } ### BEGIN ### @@ -661,7 +663,7 @@ sub emit_branch { $insn = "$name$t"; $insni = "$insn" . "i"; if ($t eq "p") { - $cast = "(long)"; + $cast = "(IMM_TYPE)"; } else { $cast = ""; } @@ -744,7 +746,7 @@ sub emit_load { $upt = "\U$t\E"; print<= (sizeof(intregs) / sizeof(struct regset))) return 0; if ((type1 == DILL_F) || (type1 == DILL_D)) { *reg1p = floatregs[regpairid].reg1; @@ -1070,8 +1075,8 @@ sub initialize { 'us', '%u', 'i', '%d', 'u', '%u', - 'l', '%ld', - 'ul', '%lu', + 'l', '%zx', + 'ul', '%zx', 'p', '%p', 'f', '%g', 'd', '%g' @@ -1084,8 +1089,8 @@ sub initialize { 'us', '%x', 'i', '%x', 'u', '%x', - 'l', '%lx', - 'ul', '%lx', + 'l', '%zx', + 'ul', '%zu', 'p', '%p', 'f', '%g', 'd', '%g' @@ -1098,8 +1103,8 @@ sub initialize { "s", "short", "u", "unsigned", "i", "int", - "l", "long", - "ul", "unsigned long", + "l", "IMM_TYPE", + "ul", "UIMM_TYPE", "p", "void *", "f", "float", "d", "double", diff --git a/thirdparty/dill/dill/virtual.c b/thirdparty/dill/dill/virtual.c index b0316b13cf..312799e61a 100644 --- a/thirdparty/dill/dill/virtual.c +++ b/thirdparty/dill/dill/virtual.c @@ -7,6 +7,7 @@ #ifndef LINUX_KERNEL_MODULE #include #include +#include #include #ifdef HAVE_MALLOC_H #include @@ -62,7 +63,7 @@ virtual_print_insn(dill_stream c, void *info_ptr, void *i) OPND(insn->opnds.a3.src2)); break; case iclass_arith3i: - printf("%si %c%d, %c%d, %ld", arith3_name[insn_code], + printf("%si %c%d, %c%d, %zu", arith3_name[insn_code], OPND(insn->opnds.a3i.dest), OPND(insn->opnds.a3i.src), insn->opnds.a3i.u.imm); break; @@ -95,7 +96,7 @@ virtual_print_insn(dill_stream c, void *info_ptr, void *i) int typ = insn->insn_code & 0xf; int store = (insn->insn_code & 0x10) == 0x10; int bswap = (insn->insn_code & 0x20) == 0x20; - printf("%s%s%si %c%d, %c%d, %ld", bswap ? "bs" : "", + printf("%s%s%si %c%d, %c%d, %zu", bswap ? "bs" : "", store == 0 ? "ld" : "st", dill_type_names[typ], OPND(insn->opnds.a3i.dest), OPND(insn->opnds.a3i.src), insn->opnds.a3i.u.imm); @@ -103,14 +104,14 @@ virtual_print_insn(dill_stream c, void *info_ptr, void *i) } case iclass_lea: { - printf("lea %c%d, %c%d, %ld", OPND(insn->opnds.a3i.dest), + printf("lea %c%d, %c%d, %zx", OPND(insn->opnds.a3i.dest), OPND(insn->opnds.a3i.src), insn->opnds.a3i.u.imm); break; } case iclass_set: { int typ = insn->insn_code & 0xf; - printf("set%s %c%d, %ld", + printf("set%s %c%d, %zx", dill_type_names[typ], OPND(insn->opnds.a3i.dest), insn->opnds.a3i.u.imm); break; @@ -139,7 +140,7 @@ virtual_print_insn(dill_stream c, void *info_ptr, void *i) case iclass_reti: { int typ = insn->insn_code & 0xf; - printf("ret%si %ld", + printf("ret%si %zu", dill_type_names[typ], insn->opnds.a3i.u.imm); break; } @@ -175,9 +176,9 @@ virtual_print_insn(dill_stream c, void *info_ptr, void *i) { int br_op = insn->insn_code; struct branch_table *t = &c->p->branch_table; - printf("b%si %c%d, %ld, L%d", branch_op_names[br_op], + printf("b%si %c%d, %p, L%d", branch_op_names[br_op], OPND(insn->opnds.bri.src), - insn->opnds.bri.imm_l, insn->opnds.bri.label); + (void*)insn->opnds.bri.imm_l, insn->opnds.bri.label); if (t->label_name[insn->opnds.bri.label] != NULL) { printf("<%s>", t->label_name[insn->opnds.bri.label]); } @@ -230,8 +231,8 @@ virtual_print_insn(dill_stream c, void *info_ptr, void *i) int reg = insn->insn_code & 0x10; if (typ != DILL_V) { if (reg != 0) { - printf("call%s R%ld, %c%d", dill_type_names[typ], - insn->opnds.calli.imm_l, OPND(insn->opnds.calli.src)); + printf("call%s R%p, %c%d", dill_type_names[typ], + (void*)insn->opnds.calli.imm_l, OPND(insn->opnds.calli.src)); } else { const char *call_name = insn->opnds.calli.xfer_name; if (call_name) { @@ -245,8 +246,8 @@ virtual_print_insn(dill_stream c, void *info_ptr, void *i) } } else { if (reg != 0) { - printf("call%s R%ld", dill_type_names[typ], - insn->opnds.calli.imm_l); + printf("call%s R%p", dill_type_names[typ], + (void*)insn->opnds.calli.imm_l); } else { const char *call_name = insn->opnds.calli.xfer_name; if (call_name) { @@ -282,7 +283,7 @@ virtual_print_insn(dill_stream c, void *info_ptr, void *i) if (typ == DILL_P) { printf("push%si 0x%p", dill_type_names[typ], insn->opnds.a3i.u.imm_a); } else { - printf("push%si 0x%lx", dill_type_names[typ], insn->opnds.a3i.u.imm); + printf("push%si 0x%zx", dill_type_names[typ], insn->opnds.a3i.u.imm); } break; } @@ -310,17 +311,13 @@ insn_same_except_dest(virtual_insn *i, virtual_insn *j) case iclass_compare: return ((icode == jcode) && (i->opnds.a3.src1 == j->opnds.a3.src1) && (i->opnds.a3.src2 == j->opnds.a3.src2)); - break; case iclass_arith3i: return ((icode == jcode) && (i->opnds.a3i.src == j->opnds.a3i.src) && (i->opnds.a3i.u.imm == j->opnds.a3i.u.imm)); - break; case iclass_arith2: return ((icode == jcode) && (i->opnds.a2.src == j->opnds.a2.src)); - break; case iclass_convert: return ((icode == jcode) && (i->opnds.a2.src == j->opnds.a2.src)); - break; case iclass_loadstore: return ((icode == jcode) && (i->opnds.a3.src1 == j->opnds.a3.src1) && (i->opnds.a3.src2 == j->opnds.a3.src2)); @@ -827,7 +824,7 @@ insn_uses(virtual_insn *insn, int *used) case iclass_call: { int reg = insn->insn_code & 0x10; if (reg != 0) { - long imm = insn->opnds.calli.imm_l; + long imm = (long) insn->opnds.calli.imm_l; int src1_vreg = imm; used[0] = src1_vreg; } @@ -1009,7 +1006,7 @@ insn_use_test(virtual_insn *insn, int vreg) case iclass_call:{ int reg = insn->insn_code & 0x10; if (reg != 0) { - long imm = insn->opnds.calli.imm_l; + long imm = (long) insn->opnds.calli.imm_l; int src1_vreg = imm; if (vreg == src1_vreg) return 1; } @@ -1179,12 +1176,11 @@ build_bbs(dill_stream c, void *vinsns, void *prefix_begin, void *code_end) { virtual_mach_info vmi = (virtual_mach_info)c->p->mach_info; basic_block bb; - int i; virtual_insn *insn, *insns = vinsns; vmi->bbcount = 0; vmi->bblist = malloc(sizeof(struct basic_block)); - i = 0; + size_t i = 0; bb = vmi->bblist; bb->start = 0; bb->label = -1; @@ -1359,7 +1355,7 @@ do_use_def_count(dill_stream c, basic_block bb, virtual_insn *insns, int loc) set_defined(c, insn->opnds.calli.src); } if (reg != 0) { - long imm = insn->opnds.calli.imm_l; + long imm = (long)insn->opnds.calli.imm_l; int src1_vreg = imm; set_used(c, src1_vreg); } @@ -1950,7 +1946,7 @@ emit_insns(dill_stream c, void *insns, label_translation_table ltable, /* arith 3 immediate operand integer insns */ int dest_vreg = ip->opnds.a3i.dest; int src_vreg = ip->opnds.a3i.src; - long imm = ip->opnds.a3i.u.imm; + intptr_t imm = ip->opnds.a3i.u.imm; int dest_preg = preg_of(c, bb, dest_vreg); int src_preg = preg_of(c, bb, src_vreg); int insn_code = ip->insn_code; @@ -2117,7 +2113,7 @@ emit_insns(dill_stream c, void *insns, label_translation_table ltable, /* load effective address */ int dest_vreg = ip->opnds.a3i.dest; int src_vreg = ip->opnds.a3i.src; - long imm = ip->opnds.a3i.u.imm; + intptr_t imm = ip->opnds.a3i.u.imm; int dest_preg = preg_of(c, bb, dest_vreg); int src_preg = preg_of(c, bb, src_vreg); if (src_preg == -1) { @@ -2154,7 +2150,7 @@ emit_insns(dill_stream c, void *insns, label_translation_table ltable, /* load store immediate operand integer insns */ int dest_vreg = ip->opnds.a3i.dest; int src_vreg = ip->opnds.a3i.src; - long imm = ip->opnds.a3i.u.imm; + intptr_t imm = ip->opnds.a3i.u.imm; int dest_preg = preg_of(c, bb, dest_vreg); int src_preg = preg_of(c, bb, src_vreg); int store = ((ip->insn_code & 0x10) == 0x10); @@ -2206,7 +2202,7 @@ emit_insns(dill_stream c, void *insns, label_translation_table ltable, { /* load store immediate operand integer insns */ int dest_vreg = ip->opnds.a3i.dest; - long imm = ip->opnds.a3i.u.imm; + intptr_t imm = ip->opnds.a3i.u.imm; int dest_preg = preg_of(c, bb, dest_vreg); int typ = ip->insn_code & 0xf; if (dest_preg == -1) { @@ -2283,7 +2279,7 @@ emit_insns(dill_stream c, void *insns, label_translation_table ltable, case iclass_reti: { /* return immediate integer insns */ - long imm = ip->opnds.a3i.u.imm; + intptr_t imm = ip->opnds.a3i.u.imm; int typ = ip->insn_code & 0xf; ( c->j->reti)(c, typ, 0, imm); last_dest_vreg = -1; @@ -2337,7 +2333,7 @@ emit_insns(dill_stream c, void *insns, label_translation_table ltable, int br_op = ip->insn_code; int label = get_new_label(ip->opnds.bri.label, ltable); int src1_vreg = ip->opnds.bri.src; - long imm = ip->opnds.bri.imm_l; + intptr_t imm = ip->opnds.bri.imm_l; int src1_preg = preg_of(c, bb, src1_vreg); if (src1_preg == -1) { /* load src1 */ @@ -2415,7 +2411,7 @@ emit_insns(dill_stream c, void *insns, label_translation_table ltable, if (typ != DILL_V) dest_preg = preg_of(c, next_bb, dest_vreg); if (reg != 0) { - int src1_vreg = ip->opnds.calli.imm_l; + intptr_t src1_vreg = ip->opnds.calli.imm_l; int src1_preg = preg_of(c, bb, src1_vreg); if (src1_preg == -1) { /* load src1 */ @@ -2495,7 +2491,7 @@ emit_insns(dill_stream c, void *insns, label_translation_table ltable, void *imm = ip->opnds.a3i.u.imm_a; dill_push_argpi(c, imm); } else { - long imm = ip->opnds.a3i.u.imm; + intptr_t imm = ip->opnds.a3i.u.imm; dill_push_argii(c, imm); } last_dest_vreg = -1; @@ -2563,7 +2559,7 @@ static void put_tentative_assigns(dill_stream c, int preg_assigned, bit_vec vec, int typ) { foreach_bit(vec, (bv_func) put_unless, c, - (void*)(long)((preg_assigned&0xffffff) | (long)typ<<24)); + (void*)(intptr_t)((preg_assigned&0xffffff) | (intptr_t)typ<<24)); } typedef struct reg_state { @@ -3180,7 +3176,7 @@ new_emit_insns(dill_stream c, void *insns, label_translation_table ltable, } break; case iclass_lea: { - int offset = ip->opnds.a3i.u.imm + offset_of(c, vused[0]); + int offset = (int) ip->opnds.a3i.u.imm + offset_of(c, vused[0]); int add_code = dill_jmp_addp; if (offset == 0) { c->j->mov(c, DILL_P, 0, pdest, dill_lp(c)); @@ -3242,7 +3238,7 @@ new_emit_insns(dill_stream c, void *insns, label_translation_table ltable, /* branch immediate */ int br_op = ip->insn_code; int label = get_new_label(ip->opnds.bri.label, ltable); - long imm = ip->opnds.bri.imm_l; + intptr_t imm = ip->opnds.bri.imm_l; ( c->j->jmp_bi)[br_op](c, c->j->b_data[br_op].data1, c->j->b_data[br_op].data2, pused[0], imm, label); @@ -3297,7 +3293,7 @@ new_emit_insns(dill_stream c, void *insns, label_translation_table ltable, void *imm = ip->opnds.a3i.u.imm_a; dill_push_argpi(c, imm); } else { - long imm = ip->opnds.a3i.u.imm; + intptr_t imm = ip->opnds.a3i.u.imm; dill_push_argii(c, imm); } break; @@ -3378,7 +3374,7 @@ const_prop_ip(dill_stream c, basic_block bb, virtual_insn *ip, virtual_insn *set int set_typ = set_ip->insn_code & 0xf; int found = 0; union { - long imm; + intptr_t imm; char *imm_a; } set; set.imm = set_ip->opnds.a3i.u.imm; @@ -3471,7 +3467,7 @@ const_prop_ip(dill_stream c, basic_block bb, virtual_insn *ip, virtual_insn *set int src_vreg = ip->opnds.a3i.src; int insn_code = ip->insn_code; union { - long imm; + intptr_t imm; char *imm_a; } u; u.imm = ip->opnds.a3i.u.imm; @@ -3806,14 +3802,17 @@ is_convert_noop(int insn_code) int to_type = insn_code & 0xf; /* GSE -bug This test should be for *generated* target, not host */ - if (sizeof(long) != sizeof(int)) return 0; - switch(from_type) { - case DILL_I: case DILL_U: - case DILL_L: case DILL_UL: - switch(to_type) { + if (sizeof(long) != sizeof(int)) { + return 0; + } else { + switch(from_type) { case DILL_I: case DILL_U: case DILL_L: case DILL_UL: - return 1; + switch(to_type) { + case DILL_I: case DILL_U: + case DILL_L: case DILL_UL: + return 1; + } } } return 0; @@ -4311,7 +4310,7 @@ extern void dill_begin_prefix_code(dill_stream s) assert(s->j->proc_start == (dill_mach_proc_start)virtual_proc_start); /* insert a return, so we don't fall into prefix code */ virtual_reti(s, DILL_I, 0, 0); - vmi->prefix_code_start = (s->p->cur_ip - s->p->code_base) / sizeof(virtual_insn); + vmi->prefix_code_start = (int) (s->p->cur_ip - s->p->code_base) / sizeof(virtual_insn); } static dill_foreign_cg_func dill_foreign_cg = NULL; @@ -4445,20 +4444,20 @@ virtual_do_end(dill_stream s, int package) } } -EXTERN void +extern void virtual_end(dill_stream c) { virtual_do_end(c, 0 /* package */); } -EXTERN void +extern void virtual_package_end(dill_stream c) { virtual_do_end(c, 1 /* package */); } -EXTERN dill_exec_ctx +extern dill_exec_ctx dill_get_exec_context(dill_stream c) { dill_exec_ctx ec = malloc(sizeof(struct dec)); @@ -4489,7 +4488,7 @@ dill_get_exec_context(dill_stream c) return ec; } -EXTERN void +extern void dill_free_exec_context(dill_exec_ctx ec) { if (ec->r) free(ec->r); @@ -4499,8 +4498,8 @@ dill_free_exec_context(dill_exec_ctx ec) free(ec); } -EXTERN void -dill_assoc_client_data(dill_exec_ctx ec, int key, long value) +extern void +dill_assoc_client_data(dill_exec_ctx ec, int key, IMM_TYPE value) { int i = 0; for (i=0; i < ec->client_data_count; i++) { @@ -4519,7 +4518,7 @@ dill_assoc_client_data(dill_exec_ctx ec, int key, long value) ec->client_data[ec->client_data_count++].value = value; } -EXTERN long +extern IMM_TYPE dill_get_client_data(dill_exec_ctx ec, int key) { int i = 0; diff --git a/thirdparty/dill/dill/virtual.h b/thirdparty/dill/dill/virtual.h index bc75e264be..910b4b3c94 100644 --- a/thirdparty/dill/dill/virtual.h +++ b/thirdparty/dill/dill/virtual.h @@ -5,7 +5,7 @@ if ((((char*)c->p->cur_ip) + sizeof(virtual_insn)) >= (char*)c->p->code_limit) { }\ *(virtual_insn*)c->p->cur_ip = i;\ if (c->dill_debug) {\ -printf("%lx -- ", (unsigned long)c->p->cur_ip);\ +printf("%p -- ", c->p->cur_ip);\ virtual_print_insn(c, NULL, c->p->cur_ip);\ printf("\n");}\ c->p->cur_ip = ((char*)c->p->cur_ip)+ sizeof(virtual_insn) diff --git a/thirdparty/dill/dill/virtual.ops b/thirdparty/dill/dill/virtual.ops index 987451ab50..bdac2355f3 100644 --- a/thirdparty/dill/dill/virtual.ops +++ b/thirdparty/dill/dill/virtual.ops @@ -33,7 +33,7 @@ sub upperc { print VMOUT "void emulate_arith3(int code, struct reg_type *dest, struct reg_type *src1, struct reg_type *src2)\n"; print VMOUT "{\n switch(code) {\n$vm_a3_code }\n}\n"; -print VMOUT "void emulate_arith3i(int code, struct reg_type *dest, struct reg_type *src1, long imm)\n"; +print VMOUT "void emulate_arith3i(int code, struct reg_type *dest, struct reg_type *src1, IMM_TYPE imm)\n"; print VMOUT "{\n switch(code) {\n$vm_a3i_code }\n}\n"; print VMOUT "void emulate_arith2(int code, struct reg_type *dest, struct reg_type *src)\n"; print VMOUT "{\n switch(code) {\n$vm_a2_code }\n}\n"; @@ -41,16 +41,16 @@ print VMOUT "int emulate_branch(int code, struct reg_type *src1, struct reg_type print VMOUT "{\n switch(code) {\n$vm_br_code }return 0;\n}\n"; print VMOUT "int emulate_compare(int code, struct reg_type *src1, struct reg_type *src2)\n"; print VMOUT "{\n switch(code) {\n$vm_c_code }return 0;\n}\n"; -print VMOUT "int emulate_branchi(int code, struct reg_type *src1, long imm)\n"; +print VMOUT "int emulate_branchi(int code, struct reg_type *src1, IMM_TYPE imm)\n"; print VMOUT "{\n switch(code) {\n$vm_bri_code }return 0;\n}\n"; print VMOUT "#define CONV(x,y) ((x<<4)+y)\n"; print VMOUT "void emulate_convert(int code, struct reg_type *dest, struct reg_type *src)\n"; print VMOUT "{\n switch(code) {\n$vm_cvt_code default: printf(\"convert missed case %x \\n\", code); break;}\n}\n"; -print VMOUT "int emulate_loadi(int code, struct reg_type *dest, struct reg_type *src, long imm)\n"; +print VMOUT "int emulate_loadi(int code, struct reg_type *dest, struct reg_type *src, IMM_TYPE imm)\n"; print VMOUT "{\n switch(code) {\n$vm_loadi_code default: printf(\"loadi missed case %x \\n\", code); break;}return 0;\n}\n"; -print VMOUT "int emulate_storei(int code, struct reg_type *dest, struct reg_type *src, long imm)\n"; +print VMOUT "int emulate_storei(int code, struct reg_type *dest, struct reg_type *src, IMM_TYPE imm)\n"; print VMOUT "{\n switch(code) {\n$vm_storei_code }return 0;\n}\n"; print COUT "DECLARE_JUMP_TABLE(${mach});\n"; @@ -147,7 +147,7 @@ EOF sub arith_insn { %src1_cast = ('p', '(char*)'); -%src2_cast = ('p', '(long)'); +%src2_cast = ('p', '(IMM_TYPE)'); local ($ops, $type_list, $subr) = @_; foreach(split(' ', $ops)) { $op = $_; @@ -214,7 +214,7 @@ sub branchi_insn { $c_op = $c_operations{$op}; foreach (split(' ', $types)) { $jmp_b_assigns = $jmp_b_assigns . "\t ${mach}_jump_table->jmp_bi[dill_jmp_b${op}${_}] = $subr;\n"; - $vm_bri_code .= "\t case dill_jmp_b${op}${_}: return ($src1_cast{$_} src1->u.${_}.${_}) $c_op $src1_cast{$_} imm; break;\n"; + $vm_bri_code .= "\t case dill_jmp_b${op}${_}: return ($src1_cast{$_} src1->u.${_}.${_}) $c_op $src1_cast{$_} imm;\n"; } } } @@ -236,8 +236,8 @@ sub compare_insn { sub convert { local($from_types, $to_types) = @_; - %src_cast = ('pul', '(long)', 'ulp', '(void*)'); - %convert_right = ('c', 'src->u.c.c', 's', '((short)(0xffff & src->u.s.s))', 'i', '((int)(0xffffffff & src->u.i.i))', 'uc', '((unsigned char)(0xff & src->u.uc.uc))', 'us', '((unsigned short)(0xffff & src->u.us.us))', 'u', '((unsigned int)(0xffffffff & src->u.u.u))', 'l', 'src->u.l.l', 'ul', '(unsigned long)src->u.ul.ul', 'd', 'src->u.d.d', 'f', 'src->u.f.f', 'p', 'src->u.l.l'); + %src_cast = ('pul', '(IMM_TYPE)', 'ulp', '(void*)'); + %convert_right = ('c', 'src->u.c.c', 's', '((short)(0xffff & src->u.s.s))', 'i', '((int)(0xffffffff & src->u.i.i))', 'uc', '((unsigned char)(0xff & src->u.uc.uc))', 'us', '((unsigned short)(0xffff & src->u.us.us))', 'u', '((unsigned int)(0xffffffff & src->u.u.u))', 'l', 'src->u.l.l', 'ul', '(UIMM_TYPE)src->u.ul.ul', 'd', 'src->u.d.d', 'f', 'src->u.f.f', 'p', 'src->u.l.l'); foreach (split(' ', $from_types)) { $from = $_; foreach (split(' ', $to_types)) { @@ -253,12 +253,12 @@ sub convert { sub load_store { local($types) = @_; - %load_store = ('c', '*((char*)(src->u.l.l + imm))', 's', '*((short*)(src->u.l.l + imm))', 'i', '*((int*)(src->u.l.l + imm))', 'uc', '*((unsigned char*)(src->u.l.l + imm))', 'us', '*((unsigned short *)(src->u.l.l + imm))', 'u', '*((unsigned int *)(src->u.l.l + imm))', 'l', '*((long*)(src->u.l.l + imm))', 'ul', '*((unsigned long *)(src->u.l.l + imm))', 'd', '*((double*)(src->u.l.l + imm))', 'f', '*((float*)(src->u.l.l + imm))', 'p', '*((void**)(src->u.l.l + imm))'); + %load_store = ('c', '*((char*)(src->u.l.l + imm))', 's', '*((short*)(src->u.l.l + imm))', 'i', '*((int*)(src->u.l.l + imm))', 'uc', '*((unsigned char*)(src->u.l.l + imm))', 'us', '*((unsigned short *)(src->u.l.l + imm))', 'u', '*((unsigned int *)(src->u.l.l + imm))', 'l', '*((IMM_TYPE*)(src->u.l.l + imm))', 'ul', '*((UIMM_TYPE *)(src->u.l.l + imm))', 'd', '*((double*)(src->u.l.l + imm))', 'f', '*((float*)(src->u.l.l + imm))', 'p', '*((void**)((char*)src->u.p.p + imm))'); foreach (split(' ', $types)) { if (($_ eq 'f') || ($_ eq 'd')) { $vm_loadi_code .= "\tcase DILL_" . &upperc(${_}). ": dest->u.${_}.${_} = $load_store{$_}; break;\n"; } else { - $vm_loadi_code .= "\tcase DILL_" . &upperc(${_}). ": dest->u.l.l = (long) $load_store{$_}; break;\n"; + $vm_loadi_code .= "\tcase DILL_" . &upperc(${_}). ": dest->u.l.l = (IMM_TYPE) $load_store{$_}; break;\n"; } $vm_storei_code .= "\tcase DILL_" . &upperc(${_}). ": $load_store{$_} = dest->u.${_}.${_}; break;\n"; } @@ -283,6 +283,7 @@ print COUT< #include +#include #include #ifdef HAVE_MALLOC_H #include @@ -333,7 +334,7 @@ virtual_compare(dill_stream s, int op, int junk, int dest, int src1, int src2) } static void -dill_varith3i(dill_stream s, int op3, int op, int dest, int src1, long imm) +dill_varith3i(dill_stream s, int op3, int op, int dest, int src1, IMM_TYPE imm) { virtual_insn i; i.class_code = iclass_arith3i; @@ -390,7 +391,7 @@ virtual_load(dill_stream s, int type, int junk, int dest, int src1, int src2) } extern void -virtual_loadi(dill_stream s, int type, int junk, int dest, int src1, long imm) +virtual_loadi(dill_stream s, int type, int junk, int dest, int src1, IMM_TYPE imm) { virtual_insn i; i.class_code = iclass_loadstorei; @@ -414,7 +415,7 @@ virtual_pbsload(dill_stream s, int type, int junk, int dest, int src1, int src2) } extern void -virtual_pbsloadi(dill_stream s, int type, int junk, int dest, int src1, long imm) +virtual_pbsloadi(dill_stream s, int type, int junk, int dest, int src1, IMM_TYPE imm) { virtual_insn i; i.class_code = iclass_loadstorei; @@ -438,7 +439,7 @@ virtual_store(dill_stream s, int type, int junk, int dest, int src1, int src2) } extern void -virtual_storei(dill_stream s, int type, int junk, int dest, int src1, long imm) +virtual_storei(dill_stream s, int type, int junk, int dest, int src1, IMM_TYPE imm) { virtual_insn i; i.class_code = iclass_loadstorei; @@ -461,7 +462,7 @@ virtual_mov(dill_stream s, int type, int junk, int dest, int src) } extern void -virtual_pset(dill_stream s, int type, int junk, int dest, long imm) +virtual_pset(dill_stream s, int type, int junk, int dest, IMM_TYPE imm) { virtual_insn i; i.class_code = iclass_set; @@ -494,7 +495,7 @@ virtual_setf(dill_stream s, int type, int junk, int dest, double imm) } extern void -virtual_reti(dill_stream s, int type, int junk, long imm) +virtual_reti(dill_stream s, int type, int junk, IMM_TYPE imm) { virtual_insn i; i.class_code = iclass_reti; @@ -527,7 +528,7 @@ virtual_mark_label(dill_stream s, int op, int type, int src1, int src2, int labe } static void -virtual_branchi(dill_stream s, int op, int type, int src, long imm, int label) +virtual_branchi(dill_stream s, int op, int type, int src, IMM_TYPE imm, int label) { virtual_insn i; i.class_code = iclass_branchi; @@ -600,7 +601,7 @@ extern void virtual_push(dill_stream s, int type, int reg) INSN_OUT(s, i); } -extern void virtual_pushi(dill_stream s, int type, long value) +extern void virtual_pushi(dill_stream s, int type, IMM_TYPE value) { virtual_insn i; i.class_code = iclass_pushi; @@ -628,7 +629,7 @@ extern void virtual_pushfi(dill_stream s, int type, double value) } extern void virtual_lea(dill_stream s, int junk1, int junk2, int dest, int src, - long imm) + IMM_TYPE imm) { virtual_insn i; i.class_code = iclass_lea; @@ -639,7 +640,7 @@ extern void virtual_lea(dill_stream s, int junk1, int junk2, int dest, int src, INSN_OUT(s, i); } -extern void virtual_special(dill_stream s, special_operations type, long param) +extern void virtual_special(dill_stream s, special_operations type, IMM_TYPE param) { virtual_insn i; i.class_code = iclass_special; @@ -662,8 +663,8 @@ int virtual_type_align[] = { 2, /* US */ 4, /* I */ 4, /* U */ - sizeof(unsigned long), /* UL */ - sizeof(long), /* L */ + sizeof(uintptr_t), /* UL */ + sizeof(intptr_t), /* L */ sizeof(char*), /* P */ 4, /* F */ 8, /* D */ @@ -679,8 +680,8 @@ int virtual_type_size[] = { 2, /* US */ 4, /* I */ 4, /* U */ - sizeof(unsigned long), /* UL */ - sizeof(long), /* L */ + sizeof(uintptr_t), /* UL */ + sizeof(intptr_t), /* L */ sizeof(char*), /* P */ sizeof(float), /* F */ sizeof(double), /* D */ diff --git a/thirdparty/dill/dill/vm.c b/thirdparty/dill/dill/vm.c index fa8a9f8fd6..a16bf4e543 100644 --- a/thirdparty/dill/dill/vm.c +++ b/thirdparty/dill/dill/vm.c @@ -4,6 +4,7 @@ #include "virtual.h" #undef NDEBUG #include "assert.h" +#include #include #ifdef HAVE_MALLOC_H #include diff --git a/thirdparty/dill/dill/vtests/CMakeLists.txt b/thirdparty/dill/dill/vtests/CMakeLists.txt index 1724f4be2a..fc3b74d67e 100644 --- a/thirdparty/dill/dill/vtests/CMakeLists.txt +++ b/thirdparty/dill/dill/vtests/CMakeLists.txt @@ -1,4 +1,3 @@ -cmake_minimum_required(VERSION 2.8) set(TESTS basic_call general t1 opt branch prefix_test) @@ -14,7 +13,7 @@ include_directories(BEFORE add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/general.c - COMMAND perl ${CMAKE_CURRENT_SOURCE_DIR}/general.ops + COMMAND ${PERL_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/general.ops MAIN_DEPENDENCY ${CMAKE_CURRENT_SOURCE_DIR}/general.ops ) @@ -24,7 +23,10 @@ foreach(TEST ${TESTS}) add_test(NAME vtest_${TEST} COMMAND v${TEST}) endforeach() -set_target_properties(vgeneral PROPERTIES LINKER_LANGUAGE C COMPILE_FLAGS "-O0") +if(NOT (CMAKE_C_COMPILER_ID MATCHES "MSVC" OR + CMAKE_C_SIMULATE_ID MATCHES "MSVC")) + set_target_properties(vgeneral PROPERTIES LINKER_LANGUAGE C COMPILE_FLAGS "-O0") +ENDIF() set_tests_properties(vtest_basic_call PROPERTIES PASS_REGULAR_EXPRESSION "########## A diff --git a/thirdparty/dill/dill/vtests/basic_call.c b/thirdparty/dill/dill/vtests/basic_call.c index c3d2a44cac..124b8d2198 100644 --- a/thirdparty/dill/dill/vtests/basic_call.c +++ b/thirdparty/dill/dill/vtests/basic_call.c @@ -117,7 +117,7 @@ void b () { if (verbose) dill_dump(s); - printf("**3=%d\n", (*(int (*)(int, int))(long)((*pp)()))(1,2)); + printf("**3=%d\n", (*(int (*)(int, int))(IMM_TYPE)((*pp)()))(1,2)); dill_free_handle(h); dill_free_stream(s); } diff --git a/thirdparty/dill/dill/vtests/general.ops b/thirdparty/dill/dill/vtests/general.ops index 2359f944dc..e7e6713762 100644 --- a/thirdparty/dill/dill/vtests/general.ops +++ b/thirdparty/dill/dill/vtests/general.ops @@ -5,12 +5,13 @@ sub upperc { return $_; } -%c_types = ('c', 'signed char', 'uc', 'unsigned char', 's', 'short', 'us', 'unsigned short', 'i', 'int', 'u', 'unsigned int', 'l', 'long', 'ul', 'unsigned long', 'p', 'char*', 'f', 'float', 'd', 'double'); +%c_types = ('c', 'signed char', 'uc', 'unsigned char', 's', 'short', 'us', 'unsigned short', 'i', 'int', 'u', 'unsigned int', 'l', 'IMM_TYPE', 'ul', 'UIMM_TYPE', 'p', 'char*', 'f', 'float', 'd', 'double'); +%c_arg_types = ('c', 'signed int', 'uc', 'unsigned int', 's', 'signed int', 'us', 'unsigned int', 'i', 'int', 'u', 'unsigned int', 'l', 'IMM_TYPE', 'ul', 'UIMM_TYPE', 'p', 'char*', 'f', 'float', 'd', 'double'); %rand_types = ('c', 'l', 'uc', 'l', 's', 'l', 'us', 'l', 'i', 'l', 'u', 'l', 'l', 'l', 'ul', 'l', 'p', '(char *)l', 'f', 'd', 'd', 'd'); -%c_print_formats = ('c', '%d', 'uc', '%u', 's', '%d', 'us', '%u', 'i', '%d', 'u', '%u', 'l', '%ld', 'ul', '%lu', 'p', '%lx', 'f', '%g', 'd', '%g'); +%c_print_formats = ('c', '%d', 'uc', '%u', 's', '%d', 'us', '%u', 'i', '%d', 'u', '%u', 'l', '%zx', 'ul', '%zu', 'p', '%p', 'f', '%g', 'd', '%g'); %drisc_arg_formats = ('c', '%i', 'uc', '%u', 's', '%i', 'us', '%u', 'i', '%i', 'u', '%u', 'l', '%l', 'ul', '%ul', 'p', '%p', 'f', '%f', 'd', '%d'); -%print_cast = ('p', '(long)'); +%print_cast = ('p', '(void*)'); $FT = "f"; $DT = "d"; @@ -88,7 +89,7 @@ sub arith_insn { $div_label = "skip$skip_count: ;\n"; $skip_count++; } elsif (($dill_op eq "rsh") || ($dill_op eq "lsh")) { - $div_continue = "if (source2_$_ >= sizeof(long)) goto skip$skip_count;"; + $div_continue = "if (source2_$_ >= sizeof(IMM_TYPE)) goto skip$skip_count;"; $div_label = "skip$skip_count: ;\n"; $skip_count++; } else { @@ -183,10 +184,10 @@ sub arithp_insn { if (defined($c_src2_values{"addl"})) { $vals = $c_src2_values{'addl'}; print COUT " for (j=0 ; j < sizeof($vals)/sizeof($vals\[0\]) ; j++) {\n"; - print COUT " long source2_l = $vals\[j\];\n"; + print COUT " IMM_TYPE source2_l = $vals\[j\];\n"; } else { print COUT " {\n"; - print COUT " long source2_l = rand2_$_;\n"; + print COUT " IMM_TYPE source2_l = rand2_$_;\n"; } # print COUT "# line ". (__LINE__ + 2) . " \"general.ops\"\n"; print COUT< +#endif -#ifdef HAVE_WINDOWS_H +#ifdef _MSC_VER +#include #define srand48(s) srand(s) -#define drand48() (((double)rand())/((double)RAND_MAX)) -#define lrand48() rand() +#define drand48() (rand()*(1./RAND_MAX)) +#define lrand48() ((long long)rand() << 32 | rand()) #define kill(x,y) #else extern double drand48(); -extern long lrand48(); -void srand48(long seedval); +extern IMM_TYPE lrand48(); +void srand48(IMM_TYPE seedval); #endif int @@ -1088,7 +1097,7 @@ EOF } print COUT " 63};\n\n"; - print COUT " unsigned long bit_pattern_vals[] = {"; + print COUT " UIMM_TYPE bit_pattern_vals[] = {"; for (0..31) { printf COUT " 0x%x, 0x%x, ",(1<<$_), ((1<<$_) - 1); } @@ -1130,6 +1139,13 @@ EOF print COUT< 1) verbose++; EOF %c_src2_values = ("rshi", "sh_src2_vals", "rshl", "sh_src2_vals", diff --git a/thirdparty/dill/dill/vtests/multi_test.c b/thirdparty/dill/dill/vtests/multi_test.c index ebdecb36ef..b8d6071cbc 100644 --- a/thirdparty/dill/dill/vtests/multi_test.c +++ b/thirdparty/dill/dill/vtests/multi_test.c @@ -1,9 +1,10 @@ -#include #include #include "dill.h" #include #include +#define assert(EX) ((EX) ? (void)0 : (fprintf(stderr, "\"%s\" failed, file %s, line %d\n", #EX, __FILE__, __LINE__), exit(1))) + int main(int argc, char **argv) { int verbose = 0; diff --git a/thirdparty/dill/dill/vtests/opt.c b/thirdparty/dill/dill/vtests/opt.c index c1c0cd51e6..ea781dabba 100644 --- a/thirdparty/dill/dill/vtests/opt.c +++ b/thirdparty/dill/dill/vtests/opt.c @@ -4,7 +4,9 @@ #include "malloc.h" #endif #include +#ifdef HAVE_UNISTD_H #include "unistd.h" +#endif #include "dill.h" #ifdef USE_MMAP_CODE_SEG diff --git a/thirdparty/dill/dill/vtests/pkg_test.c b/thirdparty/dill/dill/vtests/pkg_test.c index 6976bde94b..09912ce1f5 100644 --- a/thirdparty/dill/dill/vtests/pkg_test.c +++ b/thirdparty/dill/dill/vtests/pkg_test.c @@ -1,4 +1,3 @@ -#include #include #include "dill.h" #include diff --git a/thirdparty/dill/dill/vtests/t1.c b/thirdparty/dill/dill/vtests/t1.c index 0fa6a960a0..fb329d0b3f 100644 --- a/thirdparty/dill/dill/vtests/t1.c +++ b/thirdparty/dill/dill/vtests/t1.c @@ -4,7 +4,9 @@ #include "malloc.h" #endif #include +#ifdef HAVE_UNISTD_H #include "unistd.h" +#endif #include "dill.h" #ifdef USE_MMAP_CODE_SEG @@ -16,7 +18,7 @@ static int verbose = 0; void a () { dill_stream s = dill_create_stream(); char *target; - dill_reg a,b,p3,d,e,f,g,h,i,j,w,z; + dill_reg a,b,p3,d,e,f; dill_exec_ctx ec; dill_exec_handle handle; int (*ip)(); @@ -29,12 +31,6 @@ void a () { d = dill_getreg(s, DILL_I); e = dill_getreg(s, DILL_I); f = dill_getreg(s, DILL_I); - g = dill_getreg(s, DILL_I); - h = dill_getreg(s, DILL_I); - i = dill_getreg(s, DILL_I); - j = dill_getreg(s, DILL_I); - z = dill_getreg(s, DILL_I); - w = dill_getreg(s, DILL_I); dill_addii(s, p3, a, 5); dill_addi(s, d, a, b); @@ -59,7 +55,7 @@ void a () { #endif { int size = dill_code_size(s); - static unsigned long ps = -1; + static long ps = -1; if (ps == -1) { ps = (getpagesize ()); } diff --git a/thirdparty/dill/dill/x86_64.c b/thirdparty/dill/dill/x86_64.c index fab8cbc070..2ab77affa3 100644 --- a/thirdparty/dill/dill/x86_64.c +++ b/thirdparty/dill/dill/x86_64.c @@ -17,6 +17,7 @@ #ifndef LINUX_KERNEL_MODULE #include #include +#include #ifdef HAVE_MALLOC_H #include #endif @@ -79,14 +80,14 @@ struct basic_type_info { 2, 2, IREG}, /* US */ { 4, 4, IREG}, /* I */ { 4, 4, IREG}, /* U */ - { sizeof(long), sizeof(long), IREG}, /* UL */ - { sizeof(long), sizeof(long), IREG}, /* L */ + { sizeof(uintptr_t), sizeof(uintptr_t), IREG}, /* UL */ + { sizeof(intptr_t), sizeof(intptr_t), IREG}, /* L */ { sizeof(char*), sizeof(char*), IREG}, /* P */ { sizeof(float), sizeof(float), FREG}, /* F */ { sizeof(double), sizeof(double), FREG}, /* D */ { 0, 8, IREG}, /* V */ { -1, 8, IREG}, /* B */ - { sizeof(long), sizeof(long), IREG}, /* EC */ + { sizeof(void*), sizeof(void*), IREG}, /* EC */ }; int x86_64_type_align[] = { @@ -96,8 +97,8 @@ int x86_64_type_align[] = { 2, /* US */ 4, /* I */ 4, /* U */ - sizeof(unsigned long), /* UL */ - sizeof(long), /* L */ + sizeof(uintptr_t), /* UL */ + sizeof(intptr_t), /* L */ sizeof(char*), /* P */ 4, /* F */ 4, /* D */ @@ -113,8 +114,8 @@ int x86_64_type_size[] = { 2, /* US */ 4, /* I */ 4, /* U */ - sizeof(unsigned long), /* UL */ - sizeof(long), /* L */ + sizeof(uintptr_t), /* UL */ + sizeof(intptr_t), /* L */ sizeof(char*), /* P */ 4, /* F */ 8, /* D */ @@ -239,7 +240,7 @@ BYTE_OUT1IR(dill_stream s, int rex, int insn1, int imm32) } static void -BYTE_OUT1LR(dill_stream s, int rex, int insn1, long imm64) +BYTE_OUT1LR(dill_stream s, int rex, int insn1, intptr_t imm64) { unsigned char *tmp_ip; if (s->p->cur_ip >= s->p->code_limit) { @@ -247,12 +248,12 @@ BYTE_OUT1LR(dill_stream s, int rex, int insn1, long imm64) } tmp_ip = (unsigned char *) s->p->cur_ip; if (rex != 0) { - long tmp = imm64; + intptr_t tmp = imm64; *tmp_ip = (unsigned char)rex|0x40; *(tmp_ip + 1) = (unsigned char)insn1; memcpy(tmp_ip + 2, &tmp, 8); } else { - long tmp = imm64; + intptr_t tmp = imm64; *(tmp_ip) = (unsigned char)insn1; memcpy(tmp_ip + 1, &tmp, 8); } @@ -567,7 +568,7 @@ int src; { int rex1 = 0; if (dest > XMM7) rex1 = REX_R|REX_B; - BYTE_OUT3R(s, rex, 0x0f, 0x57, ModRM(0x3, dest, dest)); + BYTE_OUT3R(s, rex1, 0x0f, 0x57, ModRM(0x3, dest, dest)); // GSE really rex1? Late fix. } if (typ == DILL_D) op = 0xf2; BYTE_OUT1R3(s, op, rex, 0x0f, 0x5c, ModRM(0x3, dest, src)); @@ -605,7 +606,7 @@ x86_64_seti(dill_stream s, int r, int val) } static void -x86_64_setl(dill_stream s, int r, long val) +x86_64_setl(dill_stream s, int r, IMM_TYPE val) { int rex = REX_W; if (r > RDI) rex |= REX_B; @@ -617,7 +618,7 @@ x86_64_setp(dill_stream s, int type, int junk, int r, void *val) { int rex = REX_W; union { - long l; + intptr_t l; void *a; } a; a.a = val; @@ -976,7 +977,7 @@ int dest; } extern void -x86_64_ploadi(dill_stream s, int type, int junk, int dest, int src, long offset) +x86_64_ploadi(dill_stream s, int type, int junk, int dest, int src, IMM_TYPE offset) { unsigned char opcode = ld_opcodes[type]; int tmp_dest = dest; @@ -1040,7 +1041,7 @@ x86_64_ploadi(dill_stream s, int type, int junk, int dest, int src, long offset) } else { BYTE_OUT3R(s, rex, opcode, ModRM(0x0, tmp_dest, 0x4),SIB(0,4,src)); } - } else if (((long)offset <= 127) && ((long)offset > -128)) { + } else if (((intptr_t)offset <= 127) && ((intptr_t)offset > -128)) { if (float_op != 0) { BYTE_OUT1R5(s, float_op, rex, 0x0f, 0x10, ModRM(0x1, tmp_dest, src), SIB(0,ESP,0x4),offset & 0xff); } else { @@ -1060,7 +1061,7 @@ x86_64_ploadi(dill_stream s, int type, int junk, int dest, int src, long offset) } else { BYTE_OUT2R(s, rex, opcode, ModRM(0x0, tmp_dest, src)); } - } else if (((long)offset <= 127) && ((long)offset > -128)) { + } else if (((intptr_t)offset <= 127) && ((intptr_t)offset > -128)) { if (float_op != 0) { BYTE_OUT1R4(s, float_op, rex, 0x0f, 0x10, ModRM(0x1, tmp_dest, src), offset & 0xff); } else { @@ -1203,7 +1204,7 @@ x86_64_pload(dill_stream s, int type, int junk, int dest, int src1, int src2) } extern void -x86_64_pbsloadi(dill_stream s, int type, int junk, int dest, int src, long offset) +x86_64_pbsloadi(dill_stream s, int type, int junk, int dest, int src, IMM_TYPE offset) { int rex = 0; int fdest = dest; @@ -1317,7 +1318,7 @@ static unsigned char st_opcodes[] = { 0x89, /* DILL_EC */ }; extern void -x86_64_pstorei(dill_stream s, int type, int junk, int dest, int src, long offset) +x86_64_pstorei(dill_stream s, int type, int junk, int dest, int src, IMM_TYPE offset) { x86_64_mach_info smi = (x86_64_mach_info) s->p->mach_info; int rex = 0, float_op = 0; @@ -1355,7 +1356,7 @@ x86_64_pstorei(dill_stream s, int type, int junk, int dest, int src, long offset if (((src&0x7) == ESP) && (((offset & 0xffffffff80000000) == 0) || ((offset & 0xffffffff80000000) == 0xffffffff80000000))) { - if (((long)offset <= 127) && ((long)offset > -128)) { + if (((intptr_t)offset <= 127) && ((intptr_t)offset > -128)) { if (float_op != 0) { BYTE_OUT1R5(s, float_op, rex, 0x0f, 0x11, ModRM(0x1, dest, src), SIB(0x0,ESP,0x4), offset & 0xff); } else { @@ -1375,7 +1376,7 @@ x86_64_pstorei(dill_stream s, int type, int junk, int dest, int src, long offset } else { BYTE_OUT2R(s, rex, st_opcodes[type], ModRM(0x0, dest, src)); } - } else if (((long)offset <= 127) && ((long)offset > -128)) { + } else if (((intptr_t)offset <= 127) && ((intptr_t)offset > -128)) { if (float_op != 0) { BYTE_OUT1R4(s, float_op, rex, 0x0f, 0x11, ModRM(0x1, dest, src), offset & 0xff); } else { @@ -1447,7 +1448,7 @@ extern void x86_64_div(dill_stream s, int op3, int op, int dest, int src1, } extern void x86_64_divi(dill_stream s, int op3, int op, int dest, int src, - long imm) + IMM_TYPE imm) { } @@ -1586,7 +1587,7 @@ int sign; int imm; int dest; int src1; -long src2; +IMM_TYPE src2; { int rex = REX_W; /* make src1 be EAX */ @@ -1632,7 +1633,7 @@ int div; int type; int dest; int src1; -long imm; +IMM_TYPE imm; { x86_64_push_reg(s, EBP); x86_64_setl(s, EBP, imm); @@ -1719,7 +1720,7 @@ int op; int typ; int dest; int src; -long imm; +IMM_TYPE imm; { int rex = 0; if ((typ == DILL_L) || (typ == DILL_UL) || (typ == DILL_P)) { @@ -1813,7 +1814,7 @@ int op; int type; int dest; int src; -long imm; +IMM_TYPE imm; { int rex = 0; if ((type == DILL_L) || (type == DILL_UL) || (type == DILL_P)) { @@ -2125,7 +2126,7 @@ x86_64_compare(dill_stream s, int op, int type, int dest, int src1, int src2) } extern void -x86_64_comparei(dill_stream s, int op, int type, int dest, int src, long imm) +x86_64_comparei(dill_stream s, int op, int type, int dest, int src, IMM_TYPE imm) { int rex = 0; if ((type == DILL_L) || (type == DILL_UL) || (type == DILL_P)) rex |= REX_W; @@ -2172,7 +2173,7 @@ extern void x86_64_jump_to_reg(dill_stream s, unsigned long reg) extern void x86_64_jump_to_imm(dill_stream s, void *imm) { - x86_64_seti(s, EAX, (unsigned long) imm); + x86_64_seti(s, EAX, (intptr_t) imm); BYTE_OUT2(s, 0xff, ModRM(0x3, 0x4, EAX)); } @@ -2183,7 +2184,7 @@ x86_64_jal(dill_stream s, int return_addr_reg, int target) } extern void -x86_64_special(dill_stream s, special_operations type, long param) +x86_64_special(dill_stream s, special_operations type, intptr_t param) { x86_64_mach_info smi = (x86_64_mach_info) s->p->mach_info; @@ -2289,7 +2290,7 @@ static void internal_push(dill_stream s, int type, int immediate, x86_64_setl(s, EAX, a.i); arg_type = DILL_I; } else { - x86_64_setl(s, EAX, *(long*)value_ptr); + x86_64_setl(s, EAX, *(intptr_t*)value_ptr); arg_type = DILL_L; } x86_64_pstorei(s, arg_type, 0, EAX, ESP, arg.offset); @@ -2301,7 +2302,7 @@ static void internal_push(dill_stream s, int type, int immediate, } else { if ((type != DILL_F) && (type != DILL_D)) { if (arg.is_immediate) { - x86_64_setl(s, arg.out_reg, *(long*)value_ptr); + x86_64_setl(s, arg.out_reg, *(intptr_t*)value_ptr); } else { x86_64_pmov(s, arg.type, arg.out_reg, *(int*) value_ptr); } @@ -2313,7 +2314,7 @@ static void internal_push(dill_stream s, int type, int immediate, x86_64_setf(s, type, 0, arg.out_reg, *(double*)value_ptr); } else { - x86_64_setl(s, arg.out_reg, *(long*)value_ptr); + x86_64_setl(s, arg.out_reg, *(intptr_t*)value_ptr); } } else { /* move to the appropriate float reg */ @@ -2354,7 +2355,7 @@ extern void x86_64_push(dill_stream s, int type, int reg) } } -extern void x86_64_pushi(dill_stream s, int type, long value) +extern void x86_64_pushi(dill_stream s, int type, IMM_TYPE value) { internal_push(s, type, 1, &value); } @@ -2412,7 +2413,7 @@ extern int x86_64_callr(dill_stream s, int type, int src) } extern void -x86_64_branchi(dill_stream s, int op, int type, int src, long imm, int label) +x86_64_branchi(dill_stream s, int op, int type, int src, IMM_TYPE imm, int label) { int rex = 0; if ((type == DILL_L) || (type == DILL_UL) || (type == DILL_P)) rex |= REX_W; @@ -2492,7 +2493,7 @@ extern void x86_64_retf(dill_stream s, int data1, int data2, double imm) } } -extern void x86_64_reti(dill_stream s, int data1, int data2, long imm) +extern void x86_64_reti(dill_stream s, int data1, int data2, IMM_TYPE imm) { switch (data1) { case DILL_C: @@ -2583,6 +2584,12 @@ x86_64_emit_save(dill_stream s) s->p->cur_ip = save_ip; } +#ifdef USE_VIRTUAL_PROTECT +#include +#include +#include +#endif + static void x86_64_flush(void *base, void *limit) { @@ -2592,16 +2599,30 @@ x86_64_flush(void *base, void *limit) /* flush every 8 bytes of preallocated insn stream. */ while((char*)ptr < (char*) limit) { +#ifndef _MSC_VER +#ifdef __x86_64__ asm volatile ("clflush (%0)" : /* */ : "r" (ptr)); +#endif +#else + _mm_clflush(ptr); +#endif ptr = (char *)ptr + 8; } +#ifndef _MSC_VER asm volatile("nop"); asm volatile("nop"); asm volatile("nop"); asm volatile("nop"); asm volatile("nop"); +#endif } #endif +#ifdef USE_VIRTUAL_PROTECT + int result; + DWORD dummy; + size_t size = ((intptr_t)limit - (intptr_t)base); + result = VirtualProtect(base, size, PAGE_EXECUTE_READWRITE, &dummy); +#endif } extern void x86_64_end(s) @@ -2652,7 +2673,7 @@ int available_size; } extern void -x86_64_pset(dill_stream s, int type, int junk, int dest, long imm) +x86_64_pset(dill_stream s, int type, int junk, int dest, IMM_TYPE imm) { switch(type) { case DILL_L: case DILL_UL: case DILL_P: @@ -2675,7 +2696,7 @@ x86_64_setf(dill_stream s, int type, int junk, int dest, double imm) union { double d; int i[2]; - long l; + intptr_t l; } b; if (type == DILL_F) { int rex = 0; @@ -2816,7 +2837,7 @@ x86_64_count_insn(dill_stream s, int start, int end) i.buffer_length = MAXLENGTH; count = 0; insn_ptr = (char*) (i.buffer + start); - while((long)insn_ptr < (long)i.buffer + end) { + while((intptr_t)insn_ptr < (intptr_t)i.buffer + end) { insn_ptr += print_insn_i386((bfd_vma)insn_ptr, &i); count++; } diff --git a/thirdparty/dill/dill/x86_64.h b/thirdparty/dill/dill/x86_64.h index fb8da738f3..048143512b 100644 --- a/thirdparty/dill/dill/x86_64.h +++ b/thirdparty/dill/dill/x86_64.h @@ -190,53 +190,53 @@ extern int x86_64_type_size[]; extern void *gen_x86_64_mach_info(); extern void x86_64_arith3(dill_stream c, int op, int commut, int dest, int src1, int src2); extern void x86_64_arith2(dill_stream c, int op, int subop, int dest, int src); -extern void x86_64_mul(dill_stream c, int signed, int imm, int dest, int src1, long src2); +extern void x86_64_mul(dill_stream c, int signed, int imm, int dest, int src1, IMM_TYPE src2); extern void x86_64_div_mod(dill_stream c, int sign, int div, int dest, int src1, int src2); -extern void x86_64_div_modi(dill_stream c, int sign, int div, int dest, int src1, long imm); -extern void x86_64_arith3i(dill_stream c, int op, int commut, int dest, int src1, long src2); +extern void x86_64_div_modi(dill_stream c, int sign, int div, int dest, int src1, IMM_TYPE imm); +extern void x86_64_arith3i(dill_stream c, int op, int commut, int dest, int src1, IMM_TYPE src2); extern void x86_64_shift(dill_stream c, int op, int junk, int dest, int src1, int src2); -extern void x86_64_shifti(dill_stream c, int op, int junk, int dest, int src, long imm); -extern void x86_64_special(dill_stream c, special_operations type, long param); -extern void x86_64_set(dill_stream c, int r, long imm); +extern void x86_64_shifti(dill_stream c, int op, int junk, int dest, int src, IMM_TYPE imm); +extern void x86_64_special(dill_stream c, special_operations type, IMM_TYPE param); +extern void x86_64_set(dill_stream c, int r, IMM_TYPE imm); extern void x86_64_proc_start(dill_stream c, char *subr_name, int arg_count, arg_info_list args, dill_reg *arglist); extern void x86_64_end(dill_stream c); extern void x86_64_package_end(dill_stream c); extern void *x86_64_clone_code(dill_stream c, void *base, int size); extern void x86_64_ret(dill_stream c, int data1, int data2, int src); -extern void x86_64_reti(dill_stream c, int data1, int data2, long imm); +extern void x86_64_reti(dill_stream c, int data1, int data2, IMM_TYPE imm); extern void x86_64_retf(dill_stream c, int data1, int data2, double imm); extern int x86_64_getreg(dill_stream c, dill_reg *reg_p, int type, int class); extern int x86_64_putreg(dill_stream c, dill_reg reg, int type); extern void -x86_64_ploadi(dill_stream c, int type, int junk, int dest, int src, long offset); +x86_64_ploadi(dill_stream c, int type, int junk, int dest, int src, IMM_TYPE offset); extern void x86_64_pload(dill_stream c, int type, int junk, int dest, int src1, int src2); extern void -x86_64_pbsloadi(dill_stream c, int type, int junk, int dest, int src, long offset); +x86_64_pbsloadi(dill_stream c, int type, int junk, int dest, int src, IMM_TYPE offset); extern void x86_64_pbsload(dill_stream c, int type, int junk, int dest, int src1, int src2); extern void -x86_64_pstorei(dill_stream c, int type, int junk, int dest, int src, long offset); +x86_64_pstorei(dill_stream c, int type, int junk, int dest, int src, IMM_TYPE offset); extern void x86_64_pstore(dill_stream c, int type, int junk, int dest, int src1, int src2); extern void -x86_64_modi(dill_stream c, int type, int junk, int dest, int src, long offset); +x86_64_modi(dill_stream c, int type, int junk, int dest, int src, IMM_TYPE offset); extern void x86_64_mod(dill_stream c, int type, int junk, int dest, int src1, int src2); extern void -x86_64_divi(dill_stream c, int type, int junk, int dest, int src, long offset); +x86_64_divi(dill_stream c, int type, int junk, int dest, int src, IMM_TYPE offset); extern void x86_64_div(dill_stream c, int type, int junk, int dest, int src1, int src2); extern void -x86_64_converti(dill_stream c, int from_type, int to_type, int dest, long src); +x86_64_converti(dill_stream c, int from_type, int to_type, int dest, IMM_TYPE src); extern void x86_64_convert(dill_stream c, int from_type, int to_type, int dest, int src); extern void x86_64_mov(dill_stream c, int type, int junk, int dest, int src); extern void -x86_64_pset(dill_stream c, int type, int junk, int dest, long imm); +x86_64_pset(dill_stream c, int type, int junk, int dest, IMM_TYPE imm); extern void x86_64_setf(dill_stream c, int type, int junk, int dest, double imm); extern void @@ -244,13 +244,13 @@ x86_64_setp(dill_stream c, int type, int junk, int dest, void *imm); extern void x86_64_branch(dill_stream c, int op, int type, int src1, int src2, int label); extern void -x86_64_branchi(dill_stream c, int op, int type, int src, long imm, int label); +x86_64_branchi(dill_stream c, int op, int type, int src, IMM_TYPE imm, int label); extern void x86_64_compare(dill_stream s, int op, int type, int dest, int src1, int src2); extern void -x86_64_comparei(dill_stream s, int op, int type, int dest, int src, long imm); +x86_64_comparei(dill_stream s, int op, int type, int dest, int src, IMM_TYPE imm); extern void -x86_64_lea(dill_stream c, int junk, int junk1, int dest, int src, long imm); +x86_64_lea(dill_stream c, int junk, int junk1, int dest, int src, IMM_TYPE imm); extern void x86_64_farith(dill_stream c, int op, int typ, int dest, int src1, int src2); extern void @@ -263,7 +263,7 @@ extern void x86_64_jal(dill_stream c, int return_addr_reg, int target); extern int x86_64_calli(dill_stream c, int type, void *xfer_address, const char *name); extern int x86_64_callr(dill_stream c, int type, int src); extern void x86_64_push(dill_stream c, int type, int reg); -extern void x86_64_pushi(dill_stream c, int type, long value); +extern void x86_64_pushi(dill_stream c, int type, IMM_TYPE value); extern void x86_64_pushpi(dill_stream c, int type, void *value); extern void x86_64_pushfi(dill_stream c, int type, double value); extern int x86_64_local_op(dill_stream c, int flag, int val); diff --git a/thirdparty/dill/dill/x86_64_rt.c b/thirdparty/dill/dill/x86_64_rt.c index 852a407b49..9373abcba5 100644 --- a/thirdparty/dill/dill/x86_64_rt.c +++ b/thirdparty/dill/dill/x86_64_rt.c @@ -1,10 +1,19 @@ #include "config.h" +#include +#include #include "dill.h" #include "dill_internal.h" +#ifdef HAVE_SYS_MMAN_H #include "sys/mman.h" +#endif #ifdef HAVE_MEMORY_H #include "memory.h" #endif +#ifdef USE_VIRTUAL_PROTECT +#include +#include +#include +#endif #include "x86.h" extern double dill_x86_64_hidden_ULtoD(unsigned long a) @@ -23,7 +32,7 @@ x86_64_rt_call_link(char *code, call_t *t) int i; for(i=0; i< t->call_count; i++) { - unsigned long tmp = (unsigned long) t->call_locs[i].xfer_addr; + uintptr_t tmp = (uintptr_t) t->call_locs[i].xfer_addr; long *call_addr = (long *) (code + t->call_locs[i].loc + 2); memcpy(call_addr, &tmp, 8); } @@ -38,14 +47,22 @@ x86_64_flush(void *base, void *limit) /* flush every 8 bytes of preallocated insn stream. */ while((char*)ptr < (char*) limit) { +#ifndef _MSC_VER +#ifdef __x86_64__ asm volatile ("clflush (%0)" : /* */ : "r" (ptr)); +#endif +#else + _mm_clflush(ptr); +#endif ptr = (char *)ptr + 8; } +#ifndef _MSC_VER asm volatile("nop"); asm volatile("nop"); asm volatile("nop"); asm volatile("nop"); asm volatile("nop"); +#endif } #endif } @@ -65,6 +82,11 @@ x86_64_package_stitch(char *code, call_t *t, dill_pkg pkg) PROT_EXEC | PROT_READ | PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); memcpy(tmp, code, pkg->code_size); +#endif +#ifdef USE_VIRTUAL_PROTECT + int result; + DWORD dummy; + result = VirtualProtect(tmp, pkg->code_size, PAGE_EXECUTE_READWRITE, &dummy); #endif return tmp + pkg->entry_offset; }