From 6862595f56e257b8a831f9155f47536cf6667ec4 Mon Sep 17 00:00:00 2001 From: Jakub Strzebonski Date: Wed, 26 Jan 2022 11:30:05 +0100 Subject: [PATCH 01/13] #476 add LB migrating to the same node --- docs/md/lb-manager.md | 51 +++++------ src/CMakeLists.txt | 1 + .../balance/lb_invoke/lb_manager.cc | 5 ++ src/vt/vrt/collection/balance/lb_type.cc | 1 + src/vt/vrt/collection/balance/lb_type.h | 1 + .../balance/serdetestlb/serdetestlb.cc | 89 +++++++++++++++++++ .../balance/serdetestlb/serdetestlb.h | 78 ++++++++++++++++ 7 files changed, 201 insertions(+), 25 deletions(-) create mode 100644 src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc create mode 100644 src/vt/vrt/collection/balance/serdetestlb/serdetestlb.h diff --git a/docs/md/lb-manager.md b/docs/md/lb-manager.md index 1e1f176757..9f9a069c41 100644 --- a/docs/md/lb-manager.md +++ b/docs/md/lb-manager.md @@ -51,15 +51,16 @@ To print LB specification during startup, use `--vt_lb_show_spec` command line f \section load-balancers Load balancers -| Load Balancer | Type | Description | Reference | -| -------------- | ----------------------- | ---------------------------------------------- | --------- | -| RotateLB | Testing | Rotate objects in a ring | `vt::vrt::collection::lb::RotateLB` | -| RandomLB | Testing | Randomly migrate object with seed | `vt::vrt::collection::lb::RandomLB` | -| GreedyLB | Centralized | Gather to central node apply min/max heap | `vt::vrt::collection::lb::GreedyLB` | -| TemperedLB | Distributed | Inspired by epidemic algorithms | `vt::vrt::collection::lb::TemperedLB` | -| HierarchicalLB | Hierarchical | Build tree to move objects nodes | `vt::vrt::collection::lb::HierarchicalLB` | -| ZoltanLB | Hyper-graph Partitioner | Run Zoltan in hyper-graph mode to LB | `vt::vrt::collection::lb::ZoltanLB` | -| OfflineLB | User-specified | Read file to determine mapping | `vt::vrt::collection::lb::OfflineLB` | +| Load Balancer | Type | Description | Reference | +| -------------- | ----------------------- | ----------------------------------------------------------------------------------- | ----------------------------------------- | +| RotateLB | Testing | Rotate objects in a ring | `vt::vrt::collection::lb::RotateLB` | +| RandomLB | Testing | Randomly migrate object with seed | `vt::vrt::collection::lb::RandomLB` | +| GreedyLB | Centralized | Gather to central node apply min/max heap | `vt::vrt::collection::lb::GreedyLB` | +| TemperedLB | Distributed | Inspired by epidemic algorithms | `vt::vrt::collection::lb::TemperedLB` | +| HierarchicalLB | Hierarchical | Build tree to move objects nodes | `vt::vrt::collection::lb::HierarchicalLB` | +| ZoltanLB | Hyper-graph Partitioner | Run Zoltan in hyper-graph mode to LB | `vt::vrt::collection::lb::ZoltanLB` | +| OfflineLB | User-specified | Read file to determine mapping | `vt::vrt::collection::lb::OfflineLB` | +| SerdeTestLB | Testing | Migrate objects to the same node, for testing serialization/deserialization purpose | `vt::vrt::collection::lb::SerdeTestLB` | \section load-models Object Load Models @@ -104,22 +105,22 @@ times those objects will take in all future phases. The full set of load model classes provided with \vt is as follows -| Load Model | Description | Reference | -| ------------------ | --------------------------------------------------- | --------- | -| **Utilities** | | | -| LoadModel | Pure virtual interface class, which the following implement | `vt::vrt::collection::balance::LoadModel` | -| ComposedModel | A convenience class for most implementations to inherit from, that passes unmodified calls through to an underlying model instance | `vt::vrt::collection::balance::ComposedModel` | -| RawData | Returns historical data only, from the measured times | `vt::vrt::collection::balance::RawData` | -| **Transformers** | Transforms the values computed by the composed model(s), agnostic to whether a query refers to a past or future phase | | -| Norm | When asked for a `WHOLE_PHASE` value, computes a specified l-norm over all subphases | `vt::vrt::collection::balance::Norm` | -| SelectSubphases | Filters and remaps the subphases with data present in the underlying model | `vt::vrt::collection::balance::SelectSubphases` | -| CommOverhead | Adds a specified amount of imputed 'system overhead' time to each object's work based on the number of messages received | `vt::vrt::collection::balance::CommOverhead` | -| PerCollection | Maintains a set of load models associated with different collection instances, and passes queries for an object through to the model corresponding to its collection | `vt::vrt::collection::balance::PerCollection` | -| **Predictors** | Computes values for future phase queries, and passes through past phase queries | | -| NaivePersistence | Passes through historical queries, and maps all future queries to the most recent past phase | `vt::vrt::collection::balance::NaivePersistence` | -| PersistenceMedianLastN | Similar to NaivePersistence, except that it predicts based on a median over the N most recent phases | `vt::vrt::collection::balance::PersistenceMedianLastN` | -| LinearModel | Computes a linear regression over on object's loads from a number of recent phases | `vt::vrt::collection::balance::LinearModel` | -| MultiplePhases | Computes values for future phases based on sums of the underlying model's predictions for N corresponding future phases | `vt::vrt::collection::balance::MultiplePhases` | +| Load Model | Description | Reference | +| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------ | +| **Utilities** | | | +| LoadModel | Pure virtual interface class, which the following implement | `vt::vrt::collection::balance::LoadModel` | +| ComposedModel | A convenience class for most implementations to inherit from, that passes unmodified calls through to an underlying model instance | `vt::vrt::collection::balance::ComposedModel` | +| RawData | Returns historical data only, from the measured times | `vt::vrt::collection::balance::RawData` | +| **Transformers** | Transforms the values computed by the composed model(s), agnostic to whether a query refers to a past or future phase | | +| Norm | When asked for a `WHOLE_PHASE` value, computes a specified l-norm over all subphases | `vt::vrt::collection::balance::Norm` | +| SelectSubphases | Filters and remaps the subphases with data present in the underlying model | `vt::vrt::collection::balance::SelectSubphases` | +| CommOverhead | Adds a specified amount of imputed 'system overhead' time to each object's work based on the number of messages received | `vt::vrt::collection::balance::CommOverhead` | +| PerCollection | Maintains a set of load models associated with different collection instances, and passes queries for an object through to the model corresponding to its collection | `vt::vrt::collection::balance::PerCollection` | +| **Predictors** | Computes values for future phase queries, and passes through past phase queries | | +| NaivePersistence | Passes through historical queries, and maps all future queries to the most recent past phase | `vt::vrt::collection::balance::NaivePersistence` | +| PersistenceMedianLastN | Similar to NaivePersistence, except that it predicts based on a median over the N most recent phases | `vt::vrt::collection::balance::PersistenceMedianLastN` | +| LinearModel | Computes a linear regression over on object's loads from a number of recent phases | `vt::vrt::collection::balance::LinearModel` | +| MultiplePhases | Computes values for future phases based on sums of the underlying model's predictions for N corresponding future phases | `vt::vrt::collection::balance::MultiplePhases` | All of the provided load balancers described in the previous section require that the installed load model provide responses to future diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4f53a33545..e707c21054 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -89,6 +89,7 @@ set( vrt/collection/balance/offlinelb vrt/collection/balance/zoltanlb vrt/collection/balance/randomlb + vrt/collection/balance/serdetestlb vrt/collection/balance/lb_invoke vrt/collection/balance/model vrt/collection/balance/proxy diff --git a/src/vt/vrt/collection/balance/lb_invoke/lb_manager.cc b/src/vt/vrt/collection/balance/lb_invoke/lb_manager.cc index 69d2d84c32..ea30656059 100644 --- a/src/vt/vrt/collection/balance/lb_invoke/lb_manager.cc +++ b/src/vt/vrt/collection/balance/lb_invoke/lb_manager.cc @@ -58,6 +58,7 @@ #include "vt/vrt/collection/balance/lb_data_restart_reader.h" #include "vt/vrt/collection/balance/zoltanlb/zoltanlb.h" #include "vt/vrt/collection/balance/randomlb/randomlb.h" +#include "vt/vrt/collection/balance/serdetestlb/serdetestlb.h" #include "vt/vrt/collection/messages/system_create.h" #include "vt/vrt/collection/manager.fwd.h" #include "vt/utils/memory/memory_usage.h" @@ -278,6 +279,7 @@ void LBManager::startLB( # if vt_check_enabled(zoltan) case LBType::ZoltanLB: lb_instances_["chosen"] = makeLB(); break; # endif + case LBType::SerdeTestLB: lb_instances_["chosen"] = makeLB(); break; case LBType::NoLB: vtAssert(false, "LBType::NoLB is not a valid LB for collectiveImpl"); break; @@ -328,6 +330,9 @@ void LBManager::printLBArgsHelp(LBType lb) { help = lb::ZoltanLB::getInputKeysWithHelp(); break; # endif + case LBType::SerdeTestLB: + help = lb::SerdeTestLB::getInputKeysWithHelp(); + break; case LBType::NoLB: // deliberately skip retrieving arguments break; diff --git a/src/vt/vrt/collection/balance/lb_type.cc b/src/vt/vrt/collection/balance/lb_type.cc index 2889034aed..119b2fa732 100644 --- a/src/vt/vrt/collection/balance/lb_type.cc +++ b/src/vt/vrt/collection/balance/lb_type.cc @@ -60,6 +60,7 @@ static std::unordered_map lb_names_ = { {LBType::TemperedLB, std::string{"TemperedLB" }}, {LBType::OfflineLB, std::string{"OfflineLB" }}, {LBType::RandomLB, std::string{"RandomLB" }}, + {LBType::SerdeTestLB, std::string{"SerdeTestLB" }}, }; std::unordered_map& get_lb_names() { diff --git a/src/vt/vrt/collection/balance/lb_type.h b/src/vt/vrt/collection/balance/lb_type.h index 4fc67848f7..71b4f42158 100644 --- a/src/vt/vrt/collection/balance/lb_type.h +++ b/src/vt/vrt/collection/balance/lb_type.h @@ -63,6 +63,7 @@ enum struct LBType : int8_t { , ZoltanLB = 6 # endif , RandomLB = 7 + , SerdeTestLB = 8 }; }}}} /* end namespace vt::vrt::collection::balance */ diff --git a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc b/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc new file mode 100644 index 0000000000..ab97d911ee --- /dev/null +++ b/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc @@ -0,0 +1,89 @@ +/* +//@HEADER +// ***************************************************************************** +// +// serdetestlb.cc +// DARMA/vt => Virtual Transport +// +// Copyright 2019-2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +// #include "vt/config.h" +// #include "vt/timing/timing.h" +#include "vt/vrt/collection/balance/serdetestlb/serdetestlb.h" +#include "vt/vrt/collection/balance/model/load_model.h" +// #include "vt/vrt/collection/manager.h" + +// #include + +namespace vt { namespace vrt { namespace collection { namespace lb { + +/*static*/ std::unordered_map +SerdeTestLB::getInputKeysWithHelp() { + return std::unordered_map{}; +} + +void SerdeTestLB::init(objgroup::proxy::Proxy) { } + +void SerdeTestLB::inputParams(balance::SpecEntry*) { } + +void SerdeTestLB::runLB(TimeType) { + auto const this_node = theContext()->getNode(); + auto const next_node = this_node; + + if (this_node == 0) { + vt_print( + lb, + "SerdeTestLB: runLB: next_node={}\n", + next_node + ); + fflush(stdout); + } + + for (auto obj : *load_model_) { + TimeTypeWrapper const load = load_model_->getWork(obj, {balance::PhaseOffset::NEXT_PHASE, balance::PhaseOffset::WHOLE_PHASE}); + vt_debug_print( + terse, lb, + "\t SerdeTestLB::migrating object to: obj={}, load={}, to_node={} from_node={}\n", + obj, load, next_node, this_node + ); + if (obj.isMigratable()) { + migrateObjectTo(obj, next_node); + } + } +} + +}}}} /* end namespace vt::vrt::collection::lb */ diff --git a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.h b/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.h new file mode 100644 index 0000000000..78e2fc0be4 --- /dev/null +++ b/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.h @@ -0,0 +1,78 @@ +/* +//@HEADER +// ***************************************************************************** +// +// serdetestlb.h +// DARMA/vt => Virtual Transport +// +// Copyright 2019-2021 National Technology & Engineering Solutions of Sandia, LLC +// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S. +// Government retains certain rights in this software. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from this +// software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +// POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact darma@sandia.gov +// +// ***************************************************************************** +//@HEADER +*/ + +#if !defined INCLUDED_VT_VRT_COLLECTION_BALANCE_SERDETESTLB_SERDETESTLB_H +#define INCLUDED_VT_VRT_COLLECTION_BALANCE_SERDETESTLB_SERDETESTLB_H + +#include "vt/config.h" +#include "vt/messaging/message.h" +#include "vt/vrt/collection/balance/lb_common.h" +#include "vt/vrt/collection/balance/baselb/baselb.h" +#include "vt/timing/timing.h" + +#include +#include +#include +#include +#include + +namespace vt { namespace vrt { namespace collection { namespace lb { + +struct SerdeTestLB : BaseLB { + + SerdeTestLB() = default; + virtual ~SerdeTestLB() {} + + void init(objgroup::proxy::Proxy in_proxy); + void runLB(TimeType) override; + void inputParams(balance::SpecEntry* spec) override; + + static std::unordered_map getInputKeysWithHelp(); + +private: + objgroup::proxy::Proxy proxy = {}; +}; + +}}}} /* end namespace vt::vrt::collection::lb */ + +#endif /*INCLUDED_VT_VRT_COLLECTION_BALANCE_SERDETESTLB_SERDETESTLB_H*/ From a6c000139c5f6d807f3f078179a3a9936a0b9f3e Mon Sep 17 00:00:00 2001 From: Jakub Strzebonski Date: Mon, 4 Apr 2022 15:51:20 +0200 Subject: [PATCH 02/13] #476 allow migrating to the same node --- examples/collection/lb_iter.cc | 8 + src/vt/messaging/active.cc | 5 - .../vrt/collection/balance/baselb/baselb.cc | 11 +- src/vt/vrt/collection/balance/baselb/baselb.h | 3 +- .../balance/serdetestlb/serdetestlb.cc | 3 +- src/vt/vrt/collection/manager.impl.h | 196 +++++++++--------- .../migrate/migrate_handlers.impl.h | 5 +- 7 files changed, 120 insertions(+), 111 deletions(-) diff --git a/examples/collection/lb_iter.cc b/examples/collection/lb_iter.cc index 980f94bf49..0161a8b5f3 100644 --- a/examples/collection/lb_iter.cc +++ b/examples/collection/lb_iter.cc @@ -66,6 +66,14 @@ struct IterCol : vt::Collection { template void serialize(SerializerT& s) { + if (s.isSizing()) { + fmt::print("IterCol::serialize()::isSizing()\n"); + } else if (s.isPacking()) { + fmt::print("IterCol::serialize()::isPacking()\n"); + } else if (s.isUnpacking()) { + fmt::print("IterCol::serialize()::isUnpacking()\n"); + } + vt::Collection::serialize(s); s | data_2; } diff --git a/src/vt/messaging/active.cc b/src/vt/messaging/active.cc index 46f908dc9b..1316639601 100644 --- a/src/vt/messaging/active.cc +++ b/src/vt/messaging/active.cc @@ -230,11 +230,6 @@ EventType ActiveMessenger::sendMsgBytesWithPut( ); } - vtWarnIf( - !(dest != theContext()->getNode() || is_bcast), - fmt::format("Destination {} should != this node", dest) - ); - MsgSizeType new_msg_size = base.size(); if (is_put && !is_put_packed) { diff --git a/src/vt/vrt/collection/balance/baselb/baselb.cc b/src/vt/vrt/collection/balance/baselb/baselb.cc index 33348def58..8342c24116 100644 --- a/src/vt/vrt/collection/balance/baselb/baselb.cc +++ b/src/vt/vrt/collection/balance/baselb/baselb.cc @@ -142,11 +142,10 @@ std::shared_ptr BaseLB::normalizeReassignments() { auto const new_node = std::get<1>(transfer); auto const current_node = obj_id.curr_node; - // self-migration if (current_node == new_node) { - // Filter out self-migrations entirely - continue; - // vtAbort("Not currently implemented -- self-migration"); + vt_debug_print( + terse, lb, "BaseLB::normalizeReassignments(): self migration\n" + ); } // the object lives here, so it's departing. @@ -227,8 +226,8 @@ void BaseLB::notifyNewHostNodeOfObjectsArriving( } } -void BaseLB::migrateObjectTo(ObjIDType const obj_id, NodeType const to) { - if (obj_id.curr_node != to) { +void BaseLB::migrateObjectTo(ObjIDType const obj_id, NodeType const to, bool const allow_self_migration) { + if (obj_id.curr_node != to || allow_self_migration) { transfers_.push_back(TransferDestType{obj_id, to}); } } diff --git a/src/vt/vrt/collection/balance/baselb/baselb.h b/src/vt/vrt/collection/balance/baselb/baselb.h index 6616f645e1..05117b9e75 100644 --- a/src/vt/vrt/collection/balance/baselb/baselb.h +++ b/src/vt/vrt/collection/balance/baselb/baselb.h @@ -126,7 +126,8 @@ struct BaseLB { TransferVecType const& transfers, MigrationCountCB migration_count_callback ); void migrationDone(); - void migrateObjectTo(ObjIDType const obj_id, NodeType const node); + // TODO (STRZ) - in the end don't use bool var + void migrateObjectTo(ObjIDType const obj_id, NodeType const node, bool const allow_self_migration = false); void transferSend(NodeType from, TransferVecType const& transfer); void transferMigrations(TransferMsg* msg); void finalize(CountMsg* msg); diff --git a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc b/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc index ab97d911ee..3ce16eea38 100644 --- a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc +++ b/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc @@ -81,7 +81,8 @@ void SerdeTestLB::runLB(TimeType) { obj, load, next_node, this_node ); if (obj.isMigratable()) { - migrateObjectTo(obj, next_node); + constexpr bool allow_self_migration = true; + migrateObjectTo(obj, next_node, allow_self_migration); } } } diff --git a/src/vt/vrt/collection/manager.impl.h b/src/vt/vrt/collection/manager.impl.h index c71a4c8cd3..0c8496d4e4 100644 --- a/src/vt/vrt/collection/manager.impl.h +++ b/src/vt/vrt/collection/manager.impl.h @@ -1789,116 +1789,120 @@ template MigrateStatus CollectionManager::migrateOut( VirtualProxyType const& col_proxy, IndexT const& idx, NodeType const& dest ) { - auto const& this_node = theContext()->getNode(); - - vt_debug_print( - terse, vrt_coll, - "migrateOut: col_proxy={:x}, this_node={}, dest={}, " - "idx={}\n", - col_proxy, this_node, dest, print_index(idx) - ); - - if (this_node != dest) { - auto const& proxy = CollectionProxy(col_proxy).operator()( - idx - ); - auto elm_holder = findElmHolder(col_proxy); - vtAssert( - elm_holder != nullptr, "Element must be registered here" - ); + auto const& this_node = theContext()->getNode(); - #if vt_check_enabled(runtime_checks) - { - bool const exists = elm_holder->exists(idx); - vtAssert( - exists, "Local element must exist here for migration to occur" - ); - } - #endif + vt_debug_print( + terse, vrt_coll, + "migrateOut: col_proxy={:x}, this_node={}, dest={}, idx={}\n", + col_proxy, this_node, dest, print_index(idx) + ); - bool const exists = elm_holder->exists(idx); - if (!exists) { - return MigrateStatus::ElementNotLocal; - } + // if (this_node != dest) { + vt_debug_print( + terse, vrt_coll, "migrating from {} to {}\n", this_node, dest + ); - vt_debug_print( - verbose, vrt_coll, - "migrateOut: (before remove) holder numElements={}\n", - elm_holder->numElements() - ); + auto const& proxy = CollectionProxy(col_proxy).operator()( + idx + ); + auto elm_holder = findElmHolder(col_proxy); + vtAssert( + elm_holder != nullptr, "Element must be registered here" + ); - if (elm_holder->numElements() == 1 and theConfig()->vt_lb_keep_last_elm) { - vt_debug_print( - normal, vrt_coll, - "migrateOut: do not migrate last element\n" - ); - return MigrateStatus::ElementNotLocal; - } +#if vt_check_enabled(runtime_checks) + { + bool const exists = elm_holder->exists(idx); + vtAssert( + exists, "Local element must exist here for migration to occur" + ); + } +#endif - auto col_unique_ptr = elm_holder->remove(idx); - auto& typed_col_ref = *static_cast(col_unique_ptr.get()); + bool const exists = elm_holder->exists(idx); + if (!exists) { + return MigrateStatus::ElementNotLocal; + } - vt_debug_print( - verbose, vrt_coll, - "migrateOut: (after remove) holder numElements={}\n", - elm_holder->numElements() - ); + vt_debug_print( + verbose, vrt_coll, + "migrateOut: (before remove) holder numElements={}\n", + elm_holder->numElements() + ); - /* - * Invoke the virtual prelude migrate out function - */ - col_unique_ptr->preMigrateOut(); + if (elm_holder->numElements() == 1 and theConfig()->vt_lb_keep_last_elm) { + vt_debug_print( + normal, vrt_coll, + "migrateOut: do not migrate last element\n" + ); + return MigrateStatus::ElementNotLocal; + } - vt_debug_print( - verbose, vrt_coll, - "migrateOut: col_proxy={:x}, idx={}, dest={}: serializing collection elm\n", - col_proxy, print_index(idx), dest - ); + auto col_unique_ptr = elm_holder->remove(idx); + auto& typed_col_ref = *static_cast(col_unique_ptr.get()); - using MigrateMsgType = MigrateMsg; + vt_debug_print( + verbose, vrt_coll, + "migrateOut: (after remove) holder numElements={}\n", + elm_holder->numElements() + ); + + /* + * Invoke the virtual prelude migrate out function + */ + col_unique_ptr->preMigrateOut(); + + vt_debug_print( + verbose, vrt_coll, + "migrateOut: col_proxy={:x}, idx={}, dest={}: serializing collection elm\n", + col_proxy, print_index(idx), dest + ); - auto msg = makeMessage(proxy, this_node, dest, &typed_col_ref); + using MigrateMsgType = MigrateMsg; - theMsg()->sendMsg< - MigrateMsgType, MigrateHandlers::migrateInHandler - >(dest, msg); + auto msg = + makeMessage(proxy, this_node, dest, &typed_col_ref); - theLocMan()->getCollectionLM(col_proxy)->entityEmigrated(idx, dest); + theMsg()->sendMsg< + MigrateMsgType, MigrateHandlers::migrateInHandler + >(dest, msg); - /* - * Invoke the virtual epilog migrate out function - */ - col_unique_ptr->epiMigrateOut(); + theLocMan()->getCollectionLM(col_proxy)->entityEmigrated(idx, dest); - vt_debug_print( - verbose, vrt_coll, - "migrateOut: col_proxy={:x}, idx={}, dest={}: invoking destroy()\n", - col_proxy, print_index(idx), dest - ); + /* + * Invoke the virtual epilog migrate out function + */ + col_unique_ptr->epiMigrateOut(); - /* - * Invoke the virtual destroy function and then null std::unique_ptr, - * which should cause the destructor to fire - */ - col_unique_ptr->destroy(); - col_unique_ptr = nullptr; + vt_debug_print( + verbose, vrt_coll, + "migrateOut: col_proxy={:x}, idx={}, dest={}: invoking destroy()\n", + col_proxy, print_index(idx), dest + ); - auto const home_node = getMappedNode(col_proxy, idx); - elm_holder->applyListeners( - listener::ElementEventEnum::ElementMigratedOut, idx, home_node - ); + /* + * Invoke the virtual destroy function and then null std::unique_ptr, + * which should cause the destructor to fire + */ + col_unique_ptr->destroy(); + col_unique_ptr = nullptr; + + auto const home_node = getMappedNode(col_proxy, idx); + elm_holder->applyListeners( + listener::ElementEventEnum::ElementMigratedOut, idx, home_node + ); - return MigrateStatus::MigratedToRemote; - } else { - #if vt_check_enabled(runtime_checks) - vtAssert( - false, "Migration should only be called when to_node is != this_node" - ); - #else - // Do nothing - #endif - return MigrateStatus::NoMigrationNecessary; - } + return MigrateStatus::MigratedToRemote; + // } else { + // #if vt_check_enabled(runtime_checks) + // vtAssert( + // false, "Migration should only be called when to_node is != this_node" + // ); + // #else + // // Do nothing + // #endif + // return MigrateStatus::NoMigrationNecessary; + // } } template @@ -1906,10 +1910,11 @@ MigrateStatus CollectionManager::migrateIn( VirtualProxyType const& proxy, IndexT const& idx, NodeType const& from, VirtualPtrType vrt_elm_ptr ) { + auto const this_node = theContext()->getNode(); vt_debug_print( terse, vrt_coll, - "CollectionManager::migrateIn: proxy={:x}, idx={}, from={}, ptr={}\n", - proxy, print_index(idx), from, print_ptr(vrt_elm_ptr.get()) + "CollectionManager::migrateIn: proxy={:x}, idx={}, from={}, this_node={} ptr={}\n", + proxy, print_index(idx), from, this_node, print_ptr(vrt_elm_ptr.get()) ); auto vc_raw_ptr = vrt_elm_ptr.get(); @@ -1920,7 +1925,6 @@ MigrateStatus CollectionManager::migrateIn( vc_raw_ptr->preMigrateIn(); // Always update the element ID struct for LB statistic tracking - auto const& this_node = theContext()->getNode(); vrt_elm_ptr->elm_id_.curr_node = this_node; auto home_node = getMappedNode(proxy, idx); diff --git a/src/vt/vrt/collection/migrate/migrate_handlers.impl.h b/src/vt/vrt/collection/migrate/migrate_handlers.impl.h index bb68440d58..1cd2e38068 100644 --- a/src/vt/vrt/collection/migrate/migrate_handlers.impl.h +++ b/src/vt/vrt/collection/migrate/migrate_handlers.impl.h @@ -65,11 +65,12 @@ template auto const& col_proxy = full_proxy.getCollectionProxy(); auto const& elm_proxy = full_proxy.getElementProxy(); auto const& idx = elm_proxy.getIndex(); + auto const this_node = theContext()->getNode(); vt_debug_print( terse, vrt_coll, - "migrateInHandler: from_node={}, idx={}\n", - from_node, idx + "migrateInHandler: from_node={}, this_node={}, idx={}\n", + from_node, this_node, idx ); auto vc_elm_ptr = std::unique_ptr(msg->elm_); From 8a77e78f554b4e45cfea6cca3c0c5a63907f98d3 Mon Sep 17 00:00:00 2001 From: Jonathan Lifflander Date: Mon, 4 Apr 2022 14:15:39 -0700 Subject: [PATCH 03/13] #476: serial msg: fix bug sending the wrong message --- src/vt/serialization/messaging/serialized_messenger.impl.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vt/serialization/messaging/serialized_messenger.impl.h b/src/vt/serialization/messaging/serialized_messenger.impl.h index 44119c6068..fa65da93d4 100644 --- a/src/vt/serialization/messaging/serialized_messenger.impl.h +++ b/src/vt/serialization/messaging/serialized_messenger.impl.h @@ -405,9 +405,9 @@ template "serialMsgHandler: local msg: handler={}\n", typed_handler ); - auto base_msg = msg.template to(); + auto base_msg = user_msg.template to(); return messaging::PendingSend(base_msg, [=](MsgPtr in) { - runnable::makeRunnable(msg, true, typed_handler, node) + runnable::makeRunnable(user_msg, true, typed_handler, node) .withTDEpochFromMsg() .enqueue(); }); From 869e464a92569f58bf1ecbd335fbc76c6ea1af5c Mon Sep 17 00:00:00 2001 From: Jakub Strzebonski Date: Fri, 8 Apr 2022 18:58:17 +0200 Subject: [PATCH 04/13] #476 clean up changes --- src/vt/messaging/active.cc | 1 - .../vrt/collection/balance/baselb/baselb.cc | 16 +++++++++---- src/vt/vrt/collection/balance/baselb/baselb.h | 12 ++++++++-- .../balance/serdetestlb/serdetestlb.cc | 24 ++++--------------- .../balance/serdetestlb/serdetestlb.h | 4 ---- src/vt/vrt/collection/manager.impl.h | 16 ++----------- 6 files changed, 28 insertions(+), 45 deletions(-) diff --git a/src/vt/messaging/active.cc b/src/vt/messaging/active.cc index 1316639601..29f277748a 100644 --- a/src/vt/messaging/active.cc +++ b/src/vt/messaging/active.cc @@ -220,7 +220,6 @@ EventType ActiveMessenger::sendMsgBytesWithPut( auto const& is_term = envelopeIsTerm(msg->env); auto const& is_put = envelopeIsPut(msg->env); auto const& is_put_packed = envelopeIsPackedPutType(msg->env); - auto const& is_bcast = envelopeIsBcast(msg->env); if (!is_term || vt_check_enabled(print_term_msgs)) { vt_debug_print( diff --git a/src/vt/vrt/collection/balance/baselb/baselb.cc b/src/vt/vrt/collection/balance/baselb/baselb.cc index 8342c24116..6e1b0fe99d 100644 --- a/src/vt/vrt/collection/balance/baselb/baselb.cc +++ b/src/vt/vrt/collection/balance/baselb/baselb.cc @@ -144,7 +144,7 @@ std::shared_ptr BaseLB::normalizeReassignments() { if (current_node == new_node) { vt_debug_print( - terse, lb, "BaseLB::normalizeReassignments(): self migration\n" + verbose, lb, "BaseLB::normalizeReassignments(): self migration\n" ); } @@ -226,12 +226,20 @@ void BaseLB::notifyNewHostNodeOfObjectsArriving( } } -void BaseLB::migrateObjectTo(ObjIDType const obj_id, NodeType const to, bool const allow_self_migration) { - if (obj_id.curr_node != to || allow_self_migration) { - transfers_.push_back(TransferDestType{obj_id, to}); +void BaseLB::migrateObjectTo(ObjIDType const obj_id, NodeType const to) { + if (obj_id.curr_node != to) { + migrateObject(obj_id, to); } } +void BaseLB::migrateObjectToSelf(const ObjIDType obj_id) { + migrateObject(obj_id, obj_id.curr_node); +} + +void BaseLB::migrateObject(const ObjIDType obj_id, const NodeType to) { + transfers_.push_back(TransferDestType{obj_id, to}); +} + void BaseLB::finalize(CountMsg* msg) { auto global_count = msg->getVal(); if (migration_count_cb_) { diff --git a/src/vt/vrt/collection/balance/baselb/baselb.h b/src/vt/vrt/collection/balance/baselb/baselb.h index 05117b9e75..6835ccf071 100644 --- a/src/vt/vrt/collection/balance/baselb/baselb.h +++ b/src/vt/vrt/collection/balance/baselb/baselb.h @@ -126,8 +126,14 @@ struct BaseLB { TransferVecType const& transfers, MigrationCountCB migration_count_callback ); void migrationDone(); - // TODO (STRZ) - in the end don't use bool var - void migrateObjectTo(ObjIDType const obj_id, NodeType const node, bool const allow_self_migration = false); + void migrateObjectTo(ObjIDType const obj_id, NodeType const node); + + /* + * This function migrates object from and to the same node. It is used + * by SerdeTestLB for purpose of testing serialization and deserialization. + */ + void migrateObjectToSelf(ObjIDType const obj_id); + void transferSend(NodeType from, TransferVecType const& transfer); void transferMigrations(TransferMsg* msg); void finalize(CountMsg* msg); @@ -162,6 +168,8 @@ struct BaseLB { */ std::shared_ptr normalizeReassignments(); + void migrateObject(ObjIDType const obj_id, NodeType const node); + private: TransferVecType transfers_ = {}; TransferType off_node_migrate_ = {}; diff --git a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc b/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc index 3ce16eea38..46aa9085bb 100644 --- a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc +++ b/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc @@ -41,13 +41,8 @@ //@HEADER */ -// #include "vt/config.h" -// #include "vt/timing/timing.h" #include "vt/vrt/collection/balance/serdetestlb/serdetestlb.h" #include "vt/vrt/collection/balance/model/load_model.h" -// #include "vt/vrt/collection/manager.h" - -// #include namespace vt { namespace vrt { namespace collection { namespace lb { @@ -62,27 +57,16 @@ void SerdeTestLB::inputParams(balance::SpecEntry*) { } void SerdeTestLB::runLB(TimeType) { auto const this_node = theContext()->getNode(); - auto const next_node = this_node; - - if (this_node == 0) { - vt_print( - lb, - "SerdeTestLB: runLB: next_node={}\n", - next_node - ); - fflush(stdout); - } - for (auto obj : *load_model_) { TimeTypeWrapper const load = load_model_->getWork(obj, {balance::PhaseOffset::NEXT_PHASE, balance::PhaseOffset::WHOLE_PHASE}); vt_debug_print( terse, lb, - "\t SerdeTestLB::migrating object to: obj={}, load={}, to_node={} from_node={}\n", - obj, load, next_node, this_node + "\t SerdeTestLB::migrating object to: obj={}, load={}, from_node={} to_node={}\n", + obj, load, this_node, this_node ); + if (obj.isMigratable()) { - constexpr bool allow_self_migration = true; - migrateObjectTo(obj, next_node, allow_self_migration); + migrateObjectToSelf(obj); } } } diff --git a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.h b/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.h index 78e2fc0be4..a003a83ecf 100644 --- a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.h +++ b/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.h @@ -59,7 +59,6 @@ namespace vt { namespace vrt { namespace collection { namespace lb { struct SerdeTestLB : BaseLB { - SerdeTestLB() = default; virtual ~SerdeTestLB() {} @@ -68,9 +67,6 @@ struct SerdeTestLB : BaseLB { void inputParams(balance::SpecEntry* spec) override; static std::unordered_map getInputKeysWithHelp(); - -private: - objgroup::proxy::Proxy proxy = {}; }; }}}} /* end namespace vt::vrt::collection::lb */ diff --git a/src/vt/vrt/collection/manager.impl.h b/src/vt/vrt/collection/manager.impl.h index 0c8496d4e4..ff9bee327e 100644 --- a/src/vt/vrt/collection/manager.impl.h +++ b/src/vt/vrt/collection/manager.impl.h @@ -1789,7 +1789,7 @@ template MigrateStatus CollectionManager::migrateOut( VirtualProxyType const& col_proxy, IndexT const& idx, NodeType const& dest ) { - auto const& this_node = theContext()->getNode(); + auto const this_node = theContext()->getNode(); vt_debug_print( terse, vrt_coll, @@ -1797,7 +1797,6 @@ MigrateStatus CollectionManager::migrateOut( col_proxy, this_node, dest, print_index(idx) ); - // if (this_node != dest) { vt_debug_print( terse, vrt_coll, "migrating from {} to {}\n", this_node, dest ); @@ -1860,8 +1859,7 @@ MigrateStatus CollectionManager::migrateOut( using MigrateMsgType = MigrateMsg; - auto msg = - makeMessage(proxy, this_node, dest, &typed_col_ref); + auto msg = makeMessage(proxy, this_node, dest, &typed_col_ref); theMsg()->sendMsg< MigrateMsgType, MigrateHandlers::migrateInHandler @@ -1893,16 +1891,6 @@ MigrateStatus CollectionManager::migrateOut( ); return MigrateStatus::MigratedToRemote; - // } else { - // #if vt_check_enabled(runtime_checks) - // vtAssert( - // false, "Migration should only be called when to_node is != this_node" - // ); - // #else - // // Do nothing - // #endif - // return MigrateStatus::NoMigrationNecessary; - // } } template From 71c502f58bc9124e07eaf2cde72f81e4789ee371 Mon Sep 17 00:00:00 2001 From: Jakub Strzebonski Date: Fri, 8 Apr 2022 18:59:08 +0200 Subject: [PATCH 05/13] #476 add unit test --- tests/unit/collection/test_lb.extended.cc | 75 +++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/tests/unit/collection/test_lb.extended.cc b/tests/unit/collection/test_lb.extended.cc index 2bb999d136..336e461077 100644 --- a/tests/unit/collection/test_lb.extended.cc +++ b/tests/unit/collection/test_lb.extended.cc @@ -578,6 +578,81 @@ INSTANTIATE_TEST_SUITE_P( DumpUserdefinedDataExplode, TestDumpUserdefinedData, booleans ); +struct SerdeTestCol : vt::Collection { + template void serialize(SerializerT &s) { + vt::Collection::serialize(s); + + if (s.isSizing()) { + s | was_packed | was_unpacked | packed_on_node | unpacked_on_node; + return; + } + + was_packed = was_unpacked = false; + packed_on_node = unpacked_on_node = -1; + + if (s.isPacking()) { + was_packed = true; + packed_on_node = theContext()->getNode(); + } + + s | was_packed | was_unpacked | packed_on_node | unpacked_on_node; + + if (s.isUnpacking()) { + was_unpacked = true; + unpacked_on_node = theContext()->getNode(); + } + } + + bool was_packed = false; + bool was_unpacked = false; + int packed_on_node = -1; + int unpacked_on_node = -1; +}; + +using SerdeTestMsg = vt::CollectionMessage; + +void serdeColHandler(SerdeTestMsg *, SerdeTestCol *col) { + auto const cur_phase = thePhase()->getCurrentPhase(); + if (cur_phase < 2) { + return; + } + + EXPECT_TRUE(col->was_packed); + EXPECT_TRUE(col->was_unpacked); + EXPECT_EQ(col->packed_on_node, col->unpacked_on_node); +} + +void runSerdeTest() { + theConfig()->vt_lb = true; + theConfig()->vt_lb_name = "SerdeTestLB"; + if (theContext()->getNode() == 0) { + ::fmt::print("Testing LB: SerdeTestLB\n"); + } + + theCollective()->barrier(); + + auto range = Index1D{8}; + vrt::collection::CollectionProxy proxy; + + runInEpochCollective([&] { + proxy = theCollection()->constructCollective(range); + }); + + for (int phase = 0; phase < num_phases; ++phase) { + runInEpochCollective([&] { + proxy.broadcastCollective(); + }); + thePhase()->nextPhaseCollective(); + } +} + +struct TestLoadBalancerSerdeTestLB : TestParallelHarness {}; + +TEST_F(TestLoadBalancerSerdeTestLB, test_SerdeTestLB_load_balancer) { + theCollective()->barrier(); + runSerdeTest(); +} + }}}} // end namespace vt::tests::unit::lb #endif /*vt_check_enabled(lblite)*/ From bfce27308d74a24c1030eb27948d0efb1feec367 Mon Sep 17 00:00:00 2001 From: Jakub Strzebonski Date: Fri, 8 Apr 2022 19:01:44 +0200 Subject: [PATCH 06/13] #476 remove debug changes --- examples/collection/lb_iter.cc | 8 -------- 1 file changed, 8 deletions(-) diff --git a/examples/collection/lb_iter.cc b/examples/collection/lb_iter.cc index 0161a8b5f3..980f94bf49 100644 --- a/examples/collection/lb_iter.cc +++ b/examples/collection/lb_iter.cc @@ -66,14 +66,6 @@ struct IterCol : vt::Collection { template void serialize(SerializerT& s) { - if (s.isSizing()) { - fmt::print("IterCol::serialize()::isSizing()\n"); - } else if (s.isPacking()) { - fmt::print("IterCol::serialize()::isPacking()\n"); - } else if (s.isUnpacking()) { - fmt::print("IterCol::serialize()::isUnpacking()\n"); - } - vt::Collection::serialize(s); s | data_2; } From 8f6e191b6209285ddcf4fa874395b6250818eb76 Mon Sep 17 00:00:00 2001 From: Jakub Strzebonski Date: Tue, 17 May 2022 18:42:58 +0200 Subject: [PATCH 07/13] #476 add runtime flag that allows LB to migrate objs to the same node --- docs/md/lb-manager.md | 6 +- src/vt/collective/collective_ops.cc | 1 + src/vt/configs/arguments/app_config.h | 24 +-- src/vt/configs/arguments/args.cc | 32 ++-- src/vt/messaging/active.cc | 14 +- src/vt/runtime/runtime_banner.cc | 10 +- .../vrt/collection/balance/baselb/baselb.cc | 12 +- src/vt/vrt/collection/balance/baselb/baselb.h | 8 - .../balance/serdetestlb/serdetestlb.cc | 7 +- src/vt/vrt/collection/manager.impl.h | 146 +++++++++--------- 10 files changed, 139 insertions(+), 121 deletions(-) diff --git a/docs/md/lb-manager.md b/docs/md/lb-manager.md index 9f9a069c41..71ab85e4fe 100644 --- a/docs/md/lb-manager.md +++ b/docs/md/lb-manager.md @@ -13,8 +13,10 @@ To enable load balancing, the cmake flag \code{.cmake} -Dvt_lb_enabled=1 instrumentation of work and communication performed by collection elements. To run a load balancer at runtime: - - Pass `--vt_lb --vt_lb_name=` as a command line argument - - Write a LB specification file `--vt_lb --vt_lb_file_name=` + +- Pass `--vt_lb --vt_lb_name=` as a command line argument +- Write a LB specification file `--vt_lb --vt_lb_file_name=` +- One can also pass `--vt_lb_self_migration` as a command line argument to allow load balancer to migrate objects to the same node \section lb-specification-file LB Specification File diff --git a/src/vt/collective/collective_ops.cc b/src/vt/collective/collective_ops.cc index d950f992d9..967611865e 100644 --- a/src/vt/collective/collective_ops.cc +++ b/src/vt/collective/collective_ops.cc @@ -137,6 +137,7 @@ void printOverwrittens( printIfOverwritten(vt_lb_data_file); printIfOverwritten(vt_lb_data_dir_in); printIfOverwritten(vt_lb_data_file_in); + printIfOverwritten(vt_lb_self_migration); printIfOverwritten(vt_help_lb_args); printIfOverwritten(vt_no_detect_hang); printIfOverwritten(vt_print_no_progress); diff --git a/src/vt/configs/arguments/app_config.h b/src/vt/configs/arguments/app_config.h index 39f7cde1bd..4cf2fa5aae 100644 --- a/src/vt/configs/arguments/app_config.h +++ b/src/vt/configs/arguments/app_config.h @@ -135,21 +135,22 @@ struct AppConfig { bool vt_trace_event_polling = false; bool vt_trace_irecv_polling = false; - bool vt_lb = false; - bool vt_lb_show_spec = false; - bool vt_lb_quiet = false; - std::string vt_lb_file_name = ""; - std::string vt_lb_name = "NoLB"; - std::string vt_lb_args = ""; - int32_t vt_lb_interval = 1; - bool vt_lb_keep_last_elm = false; - bool vt_lb_data = false; - bool vt_lb_data_compress = true; + bool vt_lb = false; + bool vt_lb_show_spec = false; + bool vt_lb_quiet = false; + std::string vt_lb_file_name = ""; + std::string vt_lb_name = "NoLB"; + std::string vt_lb_args = ""; + int32_t vt_lb_interval = 1; + bool vt_lb_keep_last_elm = false; + bool vt_lb_data = false; + bool vt_lb_data_compress = true; std::string vt_lb_data_dir = "vt_lb_data"; std::string vt_lb_data_file = "data.%p.json"; std::string vt_lb_data_dir_in = "vt_lb_data_in"; std::string vt_lb_data_file_in = "data.%p.json"; - bool vt_help_lb_args = false; + bool vt_help_lb_args = false; + bool vt_lb_self_migration = false; bool vt_no_detect_hang = false; bool vt_print_no_progress = true; @@ -314,6 +315,7 @@ struct AppConfig { | vt_lb_data_dir_in | vt_lb_data_file_in | vt_help_lb_args + | vt_lb_self_migration | vt_no_detect_hang | vt_print_no_progress diff --git a/src/vt/configs/arguments/args.cc b/src/vt/configs/arguments/args.cc index 446b6a067e..9fb4ce6e84 100644 --- a/src/vt/configs/arguments/args.cc +++ b/src/vt/configs/arguments/args.cc @@ -475,27 +475,28 @@ void addLbArgs(CLI::App& app, AppConfig& appConfig) { auto lb_data_file = "Load balancing data output file name"; auto lb_data_dir_in = "Load balancing data input directory"; auto lb_data_file_in = "Load balancing data input file name"; + auto lb_self_migration = "Allow load balancer to migrate objects to the same node"; auto lbn = "NoLB"; auto lbi = 1; auto lbf = ""; auto lbd = "vt_lb_data"; auto lbs = "data"; auto lba = ""; - auto s = app.add_flag("--vt_lb", appConfig.vt_lb, lb); - auto t1 = app.add_flag("--vt_lb_quiet", appConfig.vt_lb_quiet, lb_quiet); - auto u = app.add_option("--vt_lb_file_name", appConfig.vt_lb_file_name, lb_file_name, lbf) - ->check(CLI::ExistingFile); - auto u1 = app.add_flag("--vt_lb_show_spec", appConfig.vt_lb_show_spec, lb_show_spec); - auto v = app.add_option("--vt_lb_name", appConfig.vt_lb_name, lb_name, lbn); - auto v1 = app.add_option("--vt_lb_args", appConfig.vt_lb_args, lb_args, lba); - auto w = app.add_option("--vt_lb_interval", appConfig.vt_lb_interval, lb_interval, lbi); - auto wl = app.add_flag("--vt_lb_keep_last_elm", appConfig.vt_lb_keep_last_elm, lb_keep_last_elm); - auto ww = app.add_flag("--vt_lb_data", appConfig.vt_lb_data, lb_data); - auto xz = app.add_flag("--vt_lb_data_compress", appConfig.vt_lb_data_compress, lb_data_comp); - auto wx = app.add_option("--vt_lb_data_dir", appConfig.vt_lb_data_dir, lb_data_dir, lbd); - auto wy = app.add_option("--vt_lb_data_file", appConfig.vt_lb_data_file, lb_data_file,lbs); - auto xx = app.add_option("--vt_lb_data_dir_in", appConfig.vt_lb_data_dir_in, lb_data_dir_in, lbd); - auto xy = app.add_option("--vt_lb_data_file_in", appConfig.vt_lb_data_file_in, lb_data_file_in,lbs); + auto s = app.add_flag("--vt_lb", appConfig.vt_lb, lb); + auto t1 = app.add_flag("--vt_lb_quiet", appConfig.vt_lb_quiet, lb_quiet); + auto u = app.add_option("--vt_lb_file_name", appConfig.vt_lb_file_name, lb_file_name, lbf)->check(CLI::ExistingFile); + auto u1 = app.add_flag("--vt_lb_show_spec", appConfig.vt_lb_show_spec, lb_show_spec); + auto v = app.add_option("--vt_lb_name", appConfig.vt_lb_name, lb_name, lbn); + auto v1 = app.add_option("--vt_lb_args", appConfig.vt_lb_args, lb_args, lba); + auto w = app.add_option("--vt_lb_interval", appConfig.vt_lb_interval, lb_interval, lbi); + auto wl = app.add_flag("--vt_lb_keep_last_elm", appConfig.vt_lb_keep_last_elm, lb_keep_last_elm); + auto ww = app.add_flag("--vt_lb_data", appConfig.vt_lb_data, lb_data); + auto xz = app.add_flag("--vt_lb_data_compress", appConfig.vt_lb_data_compress, lb_data_comp); + auto wx = app.add_option("--vt_lb_data_dir", appConfig.vt_lb_data_dir, lb_data_dir, lbd); + auto wy = app.add_option("--vt_lb_data_file", appConfig.vt_lb_data_file, lb_data_file,lbs); + auto xx = app.add_option("--vt_lb_data_dir_in", appConfig.vt_lb_data_dir_in, lb_data_dir_in, lbd); + auto xy = app.add_option("--vt_lb_data_file_in", appConfig.vt_lb_data_file_in, lb_data_file_in, lbs); + auto lbasm = app.add_option("--vt_lb_self_migration", appConfig.vt_lb_self_migration, lb_self_migration); auto debugLB = "Load Balancing"; s->group(debugLB); @@ -512,6 +513,7 @@ void addLbArgs(CLI::App& app, AppConfig& appConfig) { xx->group(debugLB); xy->group(debugLB); xz->group(debugLB); + lbasm->group(debugLB); // help options deliberately omitted from the debugLB group above so that // they appear grouped with --vt_help when --vt_help is used diff --git a/src/vt/messaging/active.cc b/src/vt/messaging/active.cc index 29f277748a..e8a6a0e46a 100644 --- a/src/vt/messaging/active.cc +++ b/src/vt/messaging/active.cc @@ -217,9 +217,10 @@ EventType ActiveMessenger::sendMsgBytesWithPut( TagType const& send_tag ) { auto msg = base.get(); - auto const& is_term = envelopeIsTerm(msg->env); - auto const& is_put = envelopeIsPut(msg->env); - auto const& is_put_packed = envelopeIsPackedPutType(msg->env); + auto const is_term = envelopeIsTerm(msg->env); + auto const is_put = envelopeIsPut(msg->env); + auto const is_put_packed = envelopeIsPackedPutType(msg->env); + auto const is_bcast = envelopeIsBcast(msg->env); if (!is_term || vt_check_enabled(print_term_msgs)) { vt_debug_print( @@ -229,6 +230,13 @@ EventType ActiveMessenger::sendMsgBytesWithPut( ); } + vtWarnIf( + dest == theContext()->getNode() && + not is_bcast && + not theConfig()->vt_lb_self_migration, + fmt::format("Destination {} should != this node", dest) + ); + MsgSizeType new_msg_size = base.size(); if (is_put && !is_put_packed) { diff --git a/src/vt/runtime/runtime_banner.cc b/src/vt/runtime/runtime_banner.cc index 0cf7a6581e..02c9e820b5 100644 --- a/src/vt/runtime/runtime_banner.cc +++ b/src/vt/runtime/runtime_banner.cc @@ -290,9 +290,15 @@ void Runtime::printStartupBanner() { if (getAppConfig()->vt_lb) { auto f9 = opt_on("--vt_lb", "Load balancing enabled"); fmt::print("{}\t{}{}", vt_pre, f9, reset); + + auto f10 = opt_on("--vt_lb_self_migration", "Self migration enabled"); + fmt::print("{}\t{}{}", vt_pre, f10, reset); + if (getAppConfig()->vt_lb_file_name != "") { - auto f12 = fmt::format("Reading LB specification from file \"{}\"", - getAppConfig()->vt_lb_file_name); + auto f12 = fmt::format( + "Reading LB specification from file \"{}\"", + getAppConfig()->vt_lb_file_name + ); auto f11 = opt_on("--vt_lb_file_name", f12); fmt::print("{}\t{}{}", vt_pre, f11, reset); diff --git a/src/vt/vrt/collection/balance/baselb/baselb.cc b/src/vt/vrt/collection/balance/baselb/baselb.cc index 6e1b0fe99d..53a198211b 100644 --- a/src/vt/vrt/collection/balance/baselb/baselb.cc +++ b/src/vt/vrt/collection/balance/baselb/baselb.cc @@ -227,19 +227,11 @@ void BaseLB::notifyNewHostNodeOfObjectsArriving( } void BaseLB::migrateObjectTo(ObjIDType const obj_id, NodeType const to) { - if (obj_id.curr_node != to) { - migrateObject(obj_id, to); + if (obj_id.curr_node != to || theConfig()->vt_lb_self_migration) { + transfers_.push_back(TransferDestType{obj_id, to}); } } -void BaseLB::migrateObjectToSelf(const ObjIDType obj_id) { - migrateObject(obj_id, obj_id.curr_node); -} - -void BaseLB::migrateObject(const ObjIDType obj_id, const NodeType to) { - transfers_.push_back(TransferDestType{obj_id, to}); -} - void BaseLB::finalize(CountMsg* msg) { auto global_count = msg->getVal(); if (migration_count_cb_) { diff --git a/src/vt/vrt/collection/balance/baselb/baselb.h b/src/vt/vrt/collection/balance/baselb/baselb.h index 6835ccf071..5ba85dac49 100644 --- a/src/vt/vrt/collection/balance/baselb/baselb.h +++ b/src/vt/vrt/collection/balance/baselb/baselb.h @@ -128,12 +128,6 @@ struct BaseLB { void migrationDone(); void migrateObjectTo(ObjIDType const obj_id, NodeType const node); - /* - * This function migrates object from and to the same node. It is used - * by SerdeTestLB for purpose of testing serialization and deserialization. - */ - void migrateObjectToSelf(ObjIDType const obj_id); - void transferSend(NodeType from, TransferVecType const& transfer); void transferMigrations(TransferMsg* msg); void finalize(CountMsg* msg); @@ -168,8 +162,6 @@ struct BaseLB { */ std::shared_ptr normalizeReassignments(); - void migrateObject(ObjIDType const obj_id, NodeType const node); - private: TransferVecType transfers_ = {}; TransferType off_node_migrate_ = {}; diff --git a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc b/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc index 46aa9085bb..6566bd6e83 100644 --- a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc +++ b/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc @@ -51,7 +51,12 @@ SerdeTestLB::getInputKeysWithHelp() { return std::unordered_map{}; } -void SerdeTestLB::init(objgroup::proxy::Proxy) { } +void SerdeTestLB::init(objgroup::proxy::Proxy) { + vtAssert( + theConfig()->vt_lb_self_migration, + "SerdeTestLB::init(): vt_lb_allow_self_migration flag must be set to use SerdeTestLB\n" + ); +} void SerdeTestLB::inputParams(balance::SpecEntry*) { } diff --git a/src/vt/vrt/collection/manager.impl.h b/src/vt/vrt/collection/manager.impl.h index ff9bee327e..d1e253e18c 100644 --- a/src/vt/vrt/collection/manager.impl.h +++ b/src/vt/vrt/collection/manager.impl.h @@ -1801,96 +1801,104 @@ MigrateStatus CollectionManager::migrateOut( terse, vrt_coll, "migrating from {} to {}\n", this_node, dest ); - auto const& proxy = CollectionProxy(col_proxy).operator()( - idx - ); - auto elm_holder = findElmHolder(col_proxy); - vtAssert( - elm_holder != nullptr, "Element must be registered here" - ); + if (this_node != dest || theConfig()->vt_lb_self_migration) { + auto const& proxy = CollectionProxy(col_proxy).operator()( + idx + ); + auto elm_holder = findElmHolder(col_proxy); + vtAssert( + elm_holder != nullptr, "Element must be registered here" + ); #if vt_check_enabled(runtime_checks) - { + { + bool const exists = elm_holder->exists(idx); + vtAssert( + exists, "Local element must exist here for migration to occur" + ); + } +#endif + bool const exists = elm_holder->exists(idx); - vtAssert( - exists, "Local element must exist here for migration to occur" + if (!exists) { + return MigrateStatus::ElementNotLocal; + } + + vt_debug_print( + verbose, vrt_coll, + "migrateOut: (before remove) holder numElements={}\n", + elm_holder->numElements() ); - } -#endif - bool const exists = elm_holder->exists(idx); - if (!exists) { - return MigrateStatus::ElementNotLocal; - } + if (elm_holder->numElements() == 1 and theConfig()->vt_lb_keep_last_elm) { + vt_debug_print( + normal, vrt_coll, + "migrateOut: do not migrate last element\n" + ); + return MigrateStatus::ElementNotLocal; + } - vt_debug_print( - verbose, vrt_coll, - "migrateOut: (before remove) holder numElements={}\n", - elm_holder->numElements() - ); + auto col_unique_ptr = elm_holder->remove(idx); + auto& typed_col_ref = *static_cast(col_unique_ptr.get()); - if (elm_holder->numElements() == 1 and theConfig()->vt_lb_keep_last_elm) { vt_debug_print( - normal, vrt_coll, - "migrateOut: do not migrate last element\n" + verbose, vrt_coll, + "migrateOut: (after remove) holder numElements={}\n", + elm_holder->numElements() ); - return MigrateStatus::ElementNotLocal; - } - auto col_unique_ptr = elm_holder->remove(idx); - auto& typed_col_ref = *static_cast(col_unique_ptr.get()); + /* + * Invoke the virtual prelude migrate out function + */ + col_unique_ptr->preMigrateOut(); - vt_debug_print( - verbose, vrt_coll, - "migrateOut: (after remove) holder numElements={}\n", - elm_holder->numElements() - ); + vt_debug_print( + verbose, vrt_coll, + "migrateOut: col_proxy={:x}, idx={}, dest={}: serializing collection elm\n", + col_proxy, print_index(idx), dest + ); - /* - * Invoke the virtual prelude migrate out function - */ - col_unique_ptr->preMigrateOut(); + using MigrateMsgType = MigrateMsg; - vt_debug_print( - verbose, vrt_coll, - "migrateOut: col_proxy={:x}, idx={}, dest={}: serializing collection elm\n", - col_proxy, print_index(idx), dest - ); + auto msg = makeMessage(proxy, this_node, dest, &typed_col_ref); - using MigrateMsgType = MigrateMsg; + theMsg()->sendMsg< + MigrateMsgType, MigrateHandlers::migrateInHandler + >(dest, msg); - auto msg = makeMessage(proxy, this_node, dest, &typed_col_ref); + theLocMan()->getCollectionLM(col_proxy)->entityEmigrated(idx, dest); - theMsg()->sendMsg< - MigrateMsgType, MigrateHandlers::migrateInHandler - >(dest, msg); + /* + * Invoke the virtual epilog migrate out function + */ + col_unique_ptr->epiMigrateOut(); - theLocMan()->getCollectionLM(col_proxy)->entityEmigrated(idx, dest); + vt_debug_print( + verbose, vrt_coll, + "migrateOut: col_proxy={:x}, idx={}, dest={}: invoking destroy()\n", + col_proxy, print_index(idx), dest + ); - /* - * Invoke the virtual epilog migrate out function - */ - col_unique_ptr->epiMigrateOut(); + /* + * Invoke the virtual destroy function and then null std::unique_ptr, + * which should cause the destructor to fire + */ + col_unique_ptr->destroy(); + col_unique_ptr = nullptr; - vt_debug_print( - verbose, vrt_coll, - "migrateOut: col_proxy={:x}, idx={}, dest={}: invoking destroy()\n", - col_proxy, print_index(idx), dest - ); + auto const home_node = getMappedNode(col_proxy, idx); + elm_holder->applyListeners( + listener::ElementEventEnum::ElementMigratedOut, idx, home_node + ); - /* - * Invoke the virtual destroy function and then null std::unique_ptr, - * which should cause the destructor to fire - */ - col_unique_ptr->destroy(); - col_unique_ptr = nullptr; - - auto const home_node = getMappedNode(col_proxy, idx); - elm_holder->applyListeners( - listener::ElementEventEnum::ElementMigratedOut, idx, home_node - ); + return MigrateStatus::MigratedToRemote; + } + +#if vt_check_enabled(runtime_checks) + vtAssert(false, "Migration should only be called when to_node is != this_node"); +#endif - return MigrateStatus::MigratedToRemote; + return MigrateStatus::NoMigrationNecessary; } template From af5e6415733bd6c6ecc6dea9b081b139e4589fef Mon Sep 17 00:00:00 2001 From: Jakub Strzebonski Date: Tue, 17 May 2022 18:58:34 +0200 Subject: [PATCH 08/13] #476 fix unit tests --- src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc | 4 ++-- tests/unit/collection/test_lb.extended.cc | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc b/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc index 6566bd6e83..76674b9324 100644 --- a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc +++ b/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc @@ -54,7 +54,7 @@ SerdeTestLB::getInputKeysWithHelp() { void SerdeTestLB::init(objgroup::proxy::Proxy) { vtAssert( theConfig()->vt_lb_self_migration, - "SerdeTestLB::init(): vt_lb_allow_self_migration flag must be set to use SerdeTestLB\n" + "SerdeTestLB::init(): vt_lb_self_migration flag must be set to use SerdeTestLB\n" ); } @@ -71,7 +71,7 @@ void SerdeTestLB::runLB(TimeType) { ); if (obj.isMigratable()) { - migrateObjectToSelf(obj); + migrateObjectTo(obj, this_node); } } } diff --git a/tests/unit/collection/test_lb.extended.cc b/tests/unit/collection/test_lb.extended.cc index 336e461077..5020453d86 100644 --- a/tests/unit/collection/test_lb.extended.cc +++ b/tests/unit/collection/test_lb.extended.cc @@ -624,6 +624,7 @@ void serdeColHandler(SerdeTestMsg *, SerdeTestCol *col) { void runSerdeTest() { theConfig()->vt_lb = true; + theConfig()->vt_lb_self_migration = true; theConfig()->vt_lb_name = "SerdeTestLB"; if (theContext()->getNode() == 0) { ::fmt::print("Testing LB: SerdeTestLB\n"); From 8be9198cd2c3c5968320b8cf2dae9508895b6217 Mon Sep 17 00:00:00 2001 From: Jakub Strzebonski Date: Thu, 19 May 2022 15:46:02 +0200 Subject: [PATCH 09/13] #476 clarify usage of vt_lb_name and vt_lb_file_name options --- docs/md/lb-manager.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/md/lb-manager.md b/docs/md/lb-manager.md index 71ab85e4fe..c2ca9d9e90 100644 --- a/docs/md/lb-manager.md +++ b/docs/md/lb-manager.md @@ -18,6 +18,8 @@ To run a load balancer at runtime: - Write a LB specification file `--vt_lb --vt_lb_file_name=` - One can also pass `--vt_lb_self_migration` as a command line argument to allow load balancer to migrate objects to the same node +Note that one should use either `--vt_lb_name` or `--vt_lb_file_name` option, not both. + \section lb-specification-file LB Specification File The LB specification file allows users to specify which load balancer along with From 7bb729a9eeb42b8e0df7b22cd0dc5ef5c1d524d7 Mon Sep 17 00:00:00 2001 From: Jakub Strzebonski Date: Thu, 19 May 2022 15:46:39 +0200 Subject: [PATCH 10/13] #476 make vt_lb_self_migration flag instead of option --- src/vt/configs/arguments/args.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vt/configs/arguments/args.cc b/src/vt/configs/arguments/args.cc index 9fb4ce6e84..58ae6fde1c 100644 --- a/src/vt/configs/arguments/args.cc +++ b/src/vt/configs/arguments/args.cc @@ -496,7 +496,7 @@ void addLbArgs(CLI::App& app, AppConfig& appConfig) { auto wy = app.add_option("--vt_lb_data_file", appConfig.vt_lb_data_file, lb_data_file,lbs); auto xx = app.add_option("--vt_lb_data_dir_in", appConfig.vt_lb_data_dir_in, lb_data_dir_in, lbd); auto xy = app.add_option("--vt_lb_data_file_in", appConfig.vt_lb_data_file_in, lb_data_file_in, lbs); - auto lbasm = app.add_option("--vt_lb_self_migration", appConfig.vt_lb_self_migration, lb_self_migration); + auto lbasm = app.add_flag("--vt_lb_self_migration", appConfig.vt_lb_self_migration, lb_self_migration); auto debugLB = "Load Balancing"; s->group(debugLB); From b5ddf726c9a76b02fa2583637d53c8fcca43915e Mon Sep 17 00:00:00 2001 From: Jakub Strzebonski Date: Thu, 19 May 2022 15:48:08 +0200 Subject: [PATCH 11/13] #476 remove redundant printing of this_node --- src/vt/vrt/collection/manager.impl.h | 7 ++++--- src/vt/vrt/collection/migrate/migrate_handlers.impl.h | 5 ++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/vt/vrt/collection/manager.impl.h b/src/vt/vrt/collection/manager.impl.h index d1e253e18c..38059dc3c6 100644 --- a/src/vt/vrt/collection/manager.impl.h +++ b/src/vt/vrt/collection/manager.impl.h @@ -1906,11 +1906,10 @@ MigrateStatus CollectionManager::migrateIn( VirtualProxyType const& proxy, IndexT const& idx, NodeType const& from, VirtualPtrType vrt_elm_ptr ) { - auto const this_node = theContext()->getNode(); vt_debug_print( terse, vrt_coll, - "CollectionManager::migrateIn: proxy={:x}, idx={}, from={}, this_node={} ptr={}\n", - proxy, print_index(idx), from, this_node, print_ptr(vrt_elm_ptr.get()) + "CollectionManager::migrateIn: proxy={:x}, idx={}, from={}, ptr={}\n", + proxy, print_index(idx), from, print_ptr(vrt_elm_ptr.get()) ); auto vc_raw_ptr = vrt_elm_ptr.get(); @@ -1920,6 +1919,8 @@ MigrateStatus CollectionManager::migrateIn( */ vc_raw_ptr->preMigrateIn(); + auto const& this_node = theContext()->getNode(); + // Always update the element ID struct for LB statistic tracking vrt_elm_ptr->elm_id_.curr_node = this_node; diff --git a/src/vt/vrt/collection/migrate/migrate_handlers.impl.h b/src/vt/vrt/collection/migrate/migrate_handlers.impl.h index 1cd2e38068..bb68440d58 100644 --- a/src/vt/vrt/collection/migrate/migrate_handlers.impl.h +++ b/src/vt/vrt/collection/migrate/migrate_handlers.impl.h @@ -65,12 +65,11 @@ template auto const& col_proxy = full_proxy.getCollectionProxy(); auto const& elm_proxy = full_proxy.getElementProxy(); auto const& idx = elm_proxy.getIndex(); - auto const this_node = theContext()->getNode(); vt_debug_print( terse, vrt_coll, - "migrateInHandler: from_node={}, this_node={}, idx={}\n", - from_node, this_node, idx + "migrateInHandler: from_node={}, idx={}\n", + from_node, idx ); auto vc_elm_ptr = std::unique_ptr(msg->elm_); From 078881853f33707526808b9ca6c724ed903fcf50 Mon Sep 17 00:00:00 2001 From: Jakub Strzebonski Date: Thu, 19 May 2022 15:48:24 +0200 Subject: [PATCH 12/13] #476 remove redundant includes --- src/vt/vrt/collection/balance/baselb/baselb.h | 1 - src/vt/vrt/collection/balance/serdetestlb/serdetestlb.h | 5 +---- src/vt/vrt/collection/manager.impl.h | 3 +-- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/vt/vrt/collection/balance/baselb/baselb.h b/src/vt/vrt/collection/balance/baselb/baselb.h index 5ba85dac49..6616f645e1 100644 --- a/src/vt/vrt/collection/balance/baselb/baselb.h +++ b/src/vt/vrt/collection/balance/baselb/baselb.h @@ -127,7 +127,6 @@ struct BaseLB { ); void migrationDone(); void migrateObjectTo(ObjIDType const obj_id, NodeType const node); - void transferSend(NodeType from, TransferVecType const& transfer); void transferMigrations(TransferMsg* msg); void finalize(CountMsg* msg); diff --git a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.h b/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.h index a003a83ecf..b5b04b3708 100644 --- a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.h +++ b/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.h @@ -50,10 +50,7 @@ #include "vt/vrt/collection/balance/baselb/baselb.h" #include "vt/timing/timing.h" -#include -#include -#include -#include +#include #include namespace vt { namespace vrt { namespace collection { namespace lb { diff --git a/src/vt/vrt/collection/manager.impl.h b/src/vt/vrt/collection/manager.impl.h index 38059dc3c6..05afadd191 100644 --- a/src/vt/vrt/collection/manager.impl.h +++ b/src/vt/vrt/collection/manager.impl.h @@ -1919,9 +1919,8 @@ MigrateStatus CollectionManager::migrateIn( */ vc_raw_ptr->preMigrateIn(); - auto const& this_node = theContext()->getNode(); - // Always update the element ID struct for LB statistic tracking + auto const& this_node = theContext()->getNode(); vrt_elm_ptr->elm_id_.curr_node = this_node; auto home_node = getMappedNode(proxy, idx); From 9b16989ea5391408f275e91cae453c2d3c5618eb Mon Sep 17 00:00:00 2001 From: Jakub Strzebonski Date: Tue, 24 May 2022 19:47:39 +0200 Subject: [PATCH 13/13] #476 rename SerdeTestLB to TestSerializationLB --- docs/md/lb-manager.md | 20 +++++++------- src/CMakeLists.txt | 2 +- .../balance/lb_invoke/lb_manager.cc | 22 ++++++++-------- src/vt/vrt/collection/balance/lb_type.cc | 18 ++++++------- src/vt/vrt/collection/balance/lb_type.h | 18 ++++++------- .../testserializationlb.cc} | 16 ++++++------ .../testserializationlb.h} | 16 ++++++------ tests/unit/collection/test_lb.extended.cc | 26 +++++++++---------- 8 files changed, 69 insertions(+), 69 deletions(-) rename src/vt/vrt/collection/balance/{serdetestlb/serdetestlb.cc => testserializationlb/testserializationlb.cc} (83%) rename src/vt/vrt/collection/balance/{serdetestlb/serdetestlb.h => testserializationlb/testserializationlb.h} (83%) diff --git a/docs/md/lb-manager.md b/docs/md/lb-manager.md index c2ca9d9e90..2cc7597dc4 100644 --- a/docs/md/lb-manager.md +++ b/docs/md/lb-manager.md @@ -55,16 +55,16 @@ To print LB specification during startup, use `--vt_lb_show_spec` command line f \section load-balancers Load balancers -| Load Balancer | Type | Description | Reference | -| -------------- | ----------------------- | ----------------------------------------------------------------------------------- | ----------------------------------------- | -| RotateLB | Testing | Rotate objects in a ring | `vt::vrt::collection::lb::RotateLB` | -| RandomLB | Testing | Randomly migrate object with seed | `vt::vrt::collection::lb::RandomLB` | -| GreedyLB | Centralized | Gather to central node apply min/max heap | `vt::vrt::collection::lb::GreedyLB` | -| TemperedLB | Distributed | Inspired by epidemic algorithms | `vt::vrt::collection::lb::TemperedLB` | -| HierarchicalLB | Hierarchical | Build tree to move objects nodes | `vt::vrt::collection::lb::HierarchicalLB` | -| ZoltanLB | Hyper-graph Partitioner | Run Zoltan in hyper-graph mode to LB | `vt::vrt::collection::lb::ZoltanLB` | -| OfflineLB | User-specified | Read file to determine mapping | `vt::vrt::collection::lb::OfflineLB` | -| SerdeTestLB | Testing | Migrate objects to the same node, for testing serialization/deserialization purpose | `vt::vrt::collection::lb::SerdeTestLB` | +| Load Balancer | Type | Description | Reference | +| ------------------- | ----------------------- | ----------------------------------------------------------------------------------- | ---------------------------------------------- | +| RotateLB | Testing | Rotate objects in a ring | `vt::vrt::collection::lb::RotateLB` | +| RandomLB | Testing | Randomly migrate object with seed | `vt::vrt::collection::lb::RandomLB` | +| GreedyLB | Centralized | Gather to central node apply min/max heap | `vt::vrt::collection::lb::GreedyLB` | +| TemperedLB | Distributed | Inspired by epidemic algorithms | `vt::vrt::collection::lb::TemperedLB` | +| HierarchicalLB | Hierarchical | Build tree to move objects nodes | `vt::vrt::collection::lb::HierarchicalLB` | +| ZoltanLB | Hyper-graph Partitioner | Run Zoltan in hyper-graph mode to LB | `vt::vrt::collection::lb::ZoltanLB` | +| OfflineLB | User-specified | Read file to determine mapping | `vt::vrt::collection::lb::OfflineLB` | +| TestSerializationLB | Testing | Migrate objects to the same node, for testing serialization/deserialization purpose | `vt::vrt::collection::lb::TestSerializationLB` | \section load-models Object Load Models diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e707c21054..8f50093f6d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -89,7 +89,7 @@ set( vrt/collection/balance/offlinelb vrt/collection/balance/zoltanlb vrt/collection/balance/randomlb - vrt/collection/balance/serdetestlb + vrt/collection/balance/testserializationlb vrt/collection/balance/lb_invoke vrt/collection/balance/model vrt/collection/balance/proxy diff --git a/src/vt/vrt/collection/balance/lb_invoke/lb_manager.cc b/src/vt/vrt/collection/balance/lb_invoke/lb_manager.cc index ea30656059..7ce7c8d8b5 100644 --- a/src/vt/vrt/collection/balance/lb_invoke/lb_manager.cc +++ b/src/vt/vrt/collection/balance/lb_invoke/lb_manager.cc @@ -58,7 +58,7 @@ #include "vt/vrt/collection/balance/lb_data_restart_reader.h" #include "vt/vrt/collection/balance/zoltanlb/zoltanlb.h" #include "vt/vrt/collection/balance/randomlb/randomlb.h" -#include "vt/vrt/collection/balance/serdetestlb/serdetestlb.h" +#include "vt/vrt/collection/balance/testserializationlb/testserializationlb.h" #include "vt/vrt/collection/messages/system_create.h" #include "vt/vrt/collection/manager.fwd.h" #include "vt/utils/memory/memory_usage.h" @@ -270,16 +270,16 @@ void LBManager::startLB( } switch (lb) { - case LBType::HierarchicalLB: lb_instances_["chosen"] = makeLB(); break; - case LBType::GreedyLB: lb_instances_["chosen"] = makeLB(); break; - case LBType::RotateLB: lb_instances_["chosen"] = makeLB(); break; - case LBType::TemperedLB: lb_instances_["chosen"] = makeLB(); break; - case LBType::OfflineLB: lb_instances_["chosen"] = makeLB(); break; - case LBType::RandomLB: lb_instances_["chosen"] = makeLB(); break; + case LBType::HierarchicalLB: lb_instances_["chosen"] = makeLB(); break; + case LBType::GreedyLB: lb_instances_["chosen"] = makeLB(); break; + case LBType::RotateLB: lb_instances_["chosen"] = makeLB(); break; + case LBType::TemperedLB: lb_instances_["chosen"] = makeLB(); break; + case LBType::OfflineLB: lb_instances_["chosen"] = makeLB(); break; + case LBType::RandomLB: lb_instances_["chosen"] = makeLB(); break; # if vt_check_enabled(zoltan) - case LBType::ZoltanLB: lb_instances_["chosen"] = makeLB(); break; + case LBType::ZoltanLB: lb_instances_["chosen"] = makeLB(); break; # endif - case LBType::SerdeTestLB: lb_instances_["chosen"] = makeLB(); break; + case LBType::TestSerializationLB: lb_instances_["chosen"] = makeLB(); break; case LBType::NoLB: vtAssert(false, "LBType::NoLB is not a valid LB for collectiveImpl"); break; @@ -330,8 +330,8 @@ void LBManager::printLBArgsHelp(LBType lb) { help = lb::ZoltanLB::getInputKeysWithHelp(); break; # endif - case LBType::SerdeTestLB: - help = lb::SerdeTestLB::getInputKeysWithHelp(); + case LBType::TestSerializationLB: + help = lb::TestSerializationLB::getInputKeysWithHelp(); break; case LBType::NoLB: // deliberately skip retrieving arguments diff --git a/src/vt/vrt/collection/balance/lb_type.cc b/src/vt/vrt/collection/balance/lb_type.cc index 119b2fa732..2cc0364a44 100644 --- a/src/vt/vrt/collection/balance/lb_type.cc +++ b/src/vt/vrt/collection/balance/lb_type.cc @@ -50,17 +50,17 @@ namespace vt { namespace vrt { namespace collection { namespace balance { static std::unordered_map lb_names_ = { - {LBType::NoLB, std::string{"NoLB" }}, + {LBType::NoLB, std::string{"NoLB" }}, # if vt_check_enabled(zoltan) - {LBType::ZoltanLB, std::string{"ZoltanLB" }}, + {LBType::ZoltanLB, std::string{"ZoltanLB" }}, # endif - {LBType::GreedyLB, std::string{"GreedyLB" }}, - {LBType::HierarchicalLB, std::string{"HierarchicalLB"}}, - {LBType::RotateLB, std::string{"RotateLB" }}, - {LBType::TemperedLB, std::string{"TemperedLB" }}, - {LBType::OfflineLB, std::string{"OfflineLB" }}, - {LBType::RandomLB, std::string{"RandomLB" }}, - {LBType::SerdeTestLB, std::string{"SerdeTestLB" }}, + {LBType::GreedyLB, std::string{"GreedyLB" }}, + {LBType::HierarchicalLB, std::string{"HierarchicalLB" }}, + {LBType::RotateLB, std::string{"RotateLB" }}, + {LBType::TemperedLB, std::string{"TemperedLB" }}, + {LBType::OfflineLB, std::string{"OfflineLB" }}, + {LBType::RandomLB, std::string{"RandomLB" }}, + {LBType::TestSerializationLB, std::string{"TestSerializationLB"}}, }; std::unordered_map& get_lb_names() { diff --git a/src/vt/vrt/collection/balance/lb_type.h b/src/vt/vrt/collection/balance/lb_type.h index 71b4f42158..8a467538b0 100644 --- a/src/vt/vrt/collection/balance/lb_type.h +++ b/src/vt/vrt/collection/balance/lb_type.h @@ -53,17 +53,17 @@ namespace vt { namespace vrt { namespace collection { namespace balance { enum struct LBType : int8_t { - NoLB = 0 - , GreedyLB = 1 - , HierarchicalLB = 2 - , RotateLB = 3 - , TemperedLB = 4 - , OfflineLB = 5 + NoLB = 0 + , GreedyLB = 1 + , HierarchicalLB = 2 + , RotateLB = 3 + , TemperedLB = 4 + , OfflineLB = 5 # if vt_check_enabled(zoltan) - , ZoltanLB = 6 + , ZoltanLB = 6 # endif - , RandomLB = 7 - , SerdeTestLB = 8 + , RandomLB = 7 + , TestSerializationLB = 8 }; }}}} /* end namespace vt::vrt::collection::balance */ diff --git a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc b/src/vt/vrt/collection/balance/testserializationlb/testserializationlb.cc similarity index 83% rename from src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc rename to src/vt/vrt/collection/balance/testserializationlb/testserializationlb.cc index 76674b9324..1a7ee6a53a 100644 --- a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc +++ b/src/vt/vrt/collection/balance/testserializationlb/testserializationlb.cc @@ -2,7 +2,7 @@ //@HEADER // ***************************************************************************** // -// serdetestlb.cc +// testserializationlb.cc // DARMA/vt => Virtual Transport // // Copyright 2019-2021 National Technology & Engineering Solutions of Sandia, LLC @@ -41,32 +41,32 @@ //@HEADER */ -#include "vt/vrt/collection/balance/serdetestlb/serdetestlb.h" +#include "vt/vrt/collection/balance/testserializationlb/testserializationlb.h" #include "vt/vrt/collection/balance/model/load_model.h" namespace vt { namespace vrt { namespace collection { namespace lb { /*static*/ std::unordered_map -SerdeTestLB::getInputKeysWithHelp() { +TestSerializationLB::getInputKeysWithHelp() { return std::unordered_map{}; } -void SerdeTestLB::init(objgroup::proxy::Proxy) { +void TestSerializationLB::init(objgroup::proxy::Proxy) { vtAssert( theConfig()->vt_lb_self_migration, - "SerdeTestLB::init(): vt_lb_self_migration flag must be set to use SerdeTestLB\n" + "TestSerializationLB::init(): vt_lb_self_migration flag must be set to use TestSerializationLB\n" ); } -void SerdeTestLB::inputParams(balance::SpecEntry*) { } +void TestSerializationLB::inputParams(balance::SpecEntry*) { } -void SerdeTestLB::runLB(TimeType) { +void TestSerializationLB::runLB(TimeType) { auto const this_node = theContext()->getNode(); for (auto obj : *load_model_) { TimeTypeWrapper const load = load_model_->getWork(obj, {balance::PhaseOffset::NEXT_PHASE, balance::PhaseOffset::WHOLE_PHASE}); vt_debug_print( terse, lb, - "\t SerdeTestLB::migrating object to: obj={}, load={}, from_node={} to_node={}\n", + "\t TestSerializationLB::migrating object to: obj={}, load={}, from_node={} to_node={}\n", obj, load, this_node, this_node ); diff --git a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.h b/src/vt/vrt/collection/balance/testserializationlb/testserializationlb.h similarity index 83% rename from src/vt/vrt/collection/balance/serdetestlb/serdetestlb.h rename to src/vt/vrt/collection/balance/testserializationlb/testserializationlb.h index b5b04b3708..f8d216439f 100644 --- a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.h +++ b/src/vt/vrt/collection/balance/testserializationlb/testserializationlb.h @@ -2,7 +2,7 @@ //@HEADER // ***************************************************************************** // -// serdetestlb.h +// testserializationlb.h // DARMA/vt => Virtual Transport // // Copyright 2019-2021 National Technology & Engineering Solutions of Sandia, LLC @@ -41,8 +41,8 @@ //@HEADER */ -#if !defined INCLUDED_VT_VRT_COLLECTION_BALANCE_SERDETESTLB_SERDETESTLB_H -#define INCLUDED_VT_VRT_COLLECTION_BALANCE_SERDETESTLB_SERDETESTLB_H +#if !defined INCLUDED_VT_VRT_COLLECTION_BALANCE_TESTSERIALIZATIONLB_TESTSERIALIZATIONLB_H +#define INCLUDED_VT_VRT_COLLECTION_BALANCE_TESTSERIALIZATIONLB_TESTSERIALIZATIONLB_H #include "vt/config.h" #include "vt/messaging/message.h" @@ -55,11 +55,11 @@ namespace vt { namespace vrt { namespace collection { namespace lb { -struct SerdeTestLB : BaseLB { - SerdeTestLB() = default; - virtual ~SerdeTestLB() {} +struct TestSerializationLB : BaseLB { + TestSerializationLB() = default; + virtual ~TestSerializationLB() {} - void init(objgroup::proxy::Proxy in_proxy); + void init(objgroup::proxy::Proxy in_proxy); void runLB(TimeType) override; void inputParams(balance::SpecEntry* spec) override; @@ -68,4 +68,4 @@ struct SerdeTestLB : BaseLB { }}}} /* end namespace vt::vrt::collection::lb */ -#endif /*INCLUDED_VT_VRT_COLLECTION_BALANCE_SERDETESTLB_SERDETESTLB_H*/ +#endif /*INCLUDED_VT_VRT_COLLECTION_BALANCE_TESTSERIALIZATIONLB_TESTSERIALIZATIONLB_H*/ diff --git a/tests/unit/collection/test_lb.extended.cc b/tests/unit/collection/test_lb.extended.cc index 5020453d86..7197298bee 100644 --- a/tests/unit/collection/test_lb.extended.cc +++ b/tests/unit/collection/test_lb.extended.cc @@ -578,9 +578,9 @@ INSTANTIATE_TEST_SUITE_P( DumpUserdefinedDataExplode, TestDumpUserdefinedData, booleans ); -struct SerdeTestCol : vt::Collection { +struct SerializationTestCol : vt::Collection { template void serialize(SerializerT &s) { - vt::Collection::serialize(s); + vt::Collection::serialize(s); if (s.isSizing()) { s | was_packed | was_unpacked | packed_on_node | unpacked_on_node; @@ -609,9 +609,9 @@ struct SerdeTestCol : vt::Collection { int unpacked_on_node = -1; }; -using SerdeTestMsg = vt::CollectionMessage; +using SerializationTestMsg = vt::CollectionMessage; -void serdeColHandler(SerdeTestMsg *, SerdeTestCol *col) { +void serializationColHandler(SerializationTestMsg *, SerializationTestCol *col) { auto const cur_phase = thePhase()->getCurrentPhase(); if (cur_phase < 2) { return; @@ -622,36 +622,36 @@ void serdeColHandler(SerdeTestMsg *, SerdeTestCol *col) { EXPECT_EQ(col->packed_on_node, col->unpacked_on_node); } -void runSerdeTest() { +void runSerializationTest() { theConfig()->vt_lb = true; theConfig()->vt_lb_self_migration = true; - theConfig()->vt_lb_name = "SerdeTestLB"; + theConfig()->vt_lb_name = "TestSerializationLB"; if (theContext()->getNode() == 0) { - ::fmt::print("Testing LB: SerdeTestLB\n"); + ::fmt::print("Testing LB: TestSerializationLB\n"); } theCollective()->barrier(); auto range = Index1D{8}; - vrt::collection::CollectionProxy proxy; + vrt::collection::CollectionProxy proxy; runInEpochCollective([&] { - proxy = theCollection()->constructCollective(range); + proxy = theCollection()->constructCollective(range); }); for (int phase = 0; phase < num_phases; ++phase) { runInEpochCollective([&] { - proxy.broadcastCollective(); + proxy.broadcastCollective(); }); thePhase()->nextPhaseCollective(); } } -struct TestLoadBalancerSerdeTestLB : TestParallelHarness {}; +struct TestLoadBalancerTestSerializationLB : TestParallelHarness {}; -TEST_F(TestLoadBalancerSerdeTestLB, test_SerdeTestLB_load_balancer) { +TEST_F(TestLoadBalancerTestSerializationLB, test_TestSerializationLB_load_balancer) { theCollective()->barrier(); - runSerdeTest(); + runSerializationTest(); } }}}} // end namespace vt::tests::unit::lb