From a6c000139c5f6d807f3f078179a3a9936a0b9f3e Mon Sep 17 00:00:00 2001 From: Jakub Strzebonski Date: Mon, 4 Apr 2022 15:51:20 +0200 Subject: [PATCH] #476 allow migrating to the same node --- examples/collection/lb_iter.cc | 8 + src/vt/messaging/active.cc | 5 - .../vrt/collection/balance/baselb/baselb.cc | 11 +- src/vt/vrt/collection/balance/baselb/baselb.h | 3 +- .../balance/serdetestlb/serdetestlb.cc | 3 +- src/vt/vrt/collection/manager.impl.h | 196 +++++++++--------- .../migrate/migrate_handlers.impl.h | 5 +- 7 files changed, 120 insertions(+), 111 deletions(-) diff --git a/examples/collection/lb_iter.cc b/examples/collection/lb_iter.cc index 980f94bf49..0161a8b5f3 100644 --- a/examples/collection/lb_iter.cc +++ b/examples/collection/lb_iter.cc @@ -66,6 +66,14 @@ struct IterCol : vt::Collection { template void serialize(SerializerT& s) { + if (s.isSizing()) { + fmt::print("IterCol::serialize()::isSizing()\n"); + } else if (s.isPacking()) { + fmt::print("IterCol::serialize()::isPacking()\n"); + } else if (s.isUnpacking()) { + fmt::print("IterCol::serialize()::isUnpacking()\n"); + } + vt::Collection::serialize(s); s | data_2; } diff --git a/src/vt/messaging/active.cc b/src/vt/messaging/active.cc index 46f908dc9b..1316639601 100644 --- a/src/vt/messaging/active.cc +++ b/src/vt/messaging/active.cc @@ -230,11 +230,6 @@ EventType ActiveMessenger::sendMsgBytesWithPut( ); } - vtWarnIf( - !(dest != theContext()->getNode() || is_bcast), - fmt::format("Destination {} should != this node", dest) - ); - MsgSizeType new_msg_size = base.size(); if (is_put && !is_put_packed) { diff --git a/src/vt/vrt/collection/balance/baselb/baselb.cc b/src/vt/vrt/collection/balance/baselb/baselb.cc index 33348def58..8342c24116 100644 --- a/src/vt/vrt/collection/balance/baselb/baselb.cc +++ b/src/vt/vrt/collection/balance/baselb/baselb.cc @@ -142,11 +142,10 @@ std::shared_ptr BaseLB::normalizeReassignments() { auto const new_node = std::get<1>(transfer); auto const current_node = obj_id.curr_node; - // self-migration if (current_node == new_node) { - // Filter out self-migrations entirely - continue; - // vtAbort("Not currently implemented -- self-migration"); + vt_debug_print( + terse, lb, "BaseLB::normalizeReassignments(): self migration\n" + ); } // the object lives here, so it's departing. @@ -227,8 +226,8 @@ void BaseLB::notifyNewHostNodeOfObjectsArriving( } } -void BaseLB::migrateObjectTo(ObjIDType const obj_id, NodeType const to) { - if (obj_id.curr_node != to) { +void BaseLB::migrateObjectTo(ObjIDType const obj_id, NodeType const to, bool const allow_self_migration) { + if (obj_id.curr_node != to || allow_self_migration) { transfers_.push_back(TransferDestType{obj_id, to}); } } diff --git a/src/vt/vrt/collection/balance/baselb/baselb.h b/src/vt/vrt/collection/balance/baselb/baselb.h index 6616f645e1..05117b9e75 100644 --- a/src/vt/vrt/collection/balance/baselb/baselb.h +++ b/src/vt/vrt/collection/balance/baselb/baselb.h @@ -126,7 +126,8 @@ struct BaseLB { TransferVecType const& transfers, MigrationCountCB migration_count_callback ); void migrationDone(); - void migrateObjectTo(ObjIDType const obj_id, NodeType const node); + // TODO (STRZ) - in the end don't use bool var + void migrateObjectTo(ObjIDType const obj_id, NodeType const node, bool const allow_self_migration = false); void transferSend(NodeType from, TransferVecType const& transfer); void transferMigrations(TransferMsg* msg); void finalize(CountMsg* msg); diff --git a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc b/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc index ab97d911ee..3ce16eea38 100644 --- a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc +++ b/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc @@ -81,7 +81,8 @@ void SerdeTestLB::runLB(TimeType) { obj, load, next_node, this_node ); if (obj.isMigratable()) { - migrateObjectTo(obj, next_node); + constexpr bool allow_self_migration = true; + migrateObjectTo(obj, next_node, allow_self_migration); } } } diff --git a/src/vt/vrt/collection/manager.impl.h b/src/vt/vrt/collection/manager.impl.h index c71a4c8cd3..0c8496d4e4 100644 --- a/src/vt/vrt/collection/manager.impl.h +++ b/src/vt/vrt/collection/manager.impl.h @@ -1789,116 +1789,120 @@ template MigrateStatus CollectionManager::migrateOut( VirtualProxyType const& col_proxy, IndexT const& idx, NodeType const& dest ) { - auto const& this_node = theContext()->getNode(); - - vt_debug_print( - terse, vrt_coll, - "migrateOut: col_proxy={:x}, this_node={}, dest={}, " - "idx={}\n", - col_proxy, this_node, dest, print_index(idx) - ); - - if (this_node != dest) { - auto const& proxy = CollectionProxy(col_proxy).operator()( - idx - ); - auto elm_holder = findElmHolder(col_proxy); - vtAssert( - elm_holder != nullptr, "Element must be registered here" - ); + auto const& this_node = theContext()->getNode(); - #if vt_check_enabled(runtime_checks) - { - bool const exists = elm_holder->exists(idx); - vtAssert( - exists, "Local element must exist here for migration to occur" - ); - } - #endif + vt_debug_print( + terse, vrt_coll, + "migrateOut: col_proxy={:x}, this_node={}, dest={}, idx={}\n", + col_proxy, this_node, dest, print_index(idx) + ); - bool const exists = elm_holder->exists(idx); - if (!exists) { - return MigrateStatus::ElementNotLocal; - } + // if (this_node != dest) { + vt_debug_print( + terse, vrt_coll, "migrating from {} to {}\n", this_node, dest + ); - vt_debug_print( - verbose, vrt_coll, - "migrateOut: (before remove) holder numElements={}\n", - elm_holder->numElements() - ); + auto const& proxy = CollectionProxy(col_proxy).operator()( + idx + ); + auto elm_holder = findElmHolder(col_proxy); + vtAssert( + elm_holder != nullptr, "Element must be registered here" + ); - if (elm_holder->numElements() == 1 and theConfig()->vt_lb_keep_last_elm) { - vt_debug_print( - normal, vrt_coll, - "migrateOut: do not migrate last element\n" - ); - return MigrateStatus::ElementNotLocal; - } +#if vt_check_enabled(runtime_checks) + { + bool const exists = elm_holder->exists(idx); + vtAssert( + exists, "Local element must exist here for migration to occur" + ); + } +#endif - auto col_unique_ptr = elm_holder->remove(idx); - auto& typed_col_ref = *static_cast(col_unique_ptr.get()); + bool const exists = elm_holder->exists(idx); + if (!exists) { + return MigrateStatus::ElementNotLocal; + } - vt_debug_print( - verbose, vrt_coll, - "migrateOut: (after remove) holder numElements={}\n", - elm_holder->numElements() - ); + vt_debug_print( + verbose, vrt_coll, + "migrateOut: (before remove) holder numElements={}\n", + elm_holder->numElements() + ); - /* - * Invoke the virtual prelude migrate out function - */ - col_unique_ptr->preMigrateOut(); + if (elm_holder->numElements() == 1 and theConfig()->vt_lb_keep_last_elm) { + vt_debug_print( + normal, vrt_coll, + "migrateOut: do not migrate last element\n" + ); + return MigrateStatus::ElementNotLocal; + } - vt_debug_print( - verbose, vrt_coll, - "migrateOut: col_proxy={:x}, idx={}, dest={}: serializing collection elm\n", - col_proxy, print_index(idx), dest - ); + auto col_unique_ptr = elm_holder->remove(idx); + auto& typed_col_ref = *static_cast(col_unique_ptr.get()); - using MigrateMsgType = MigrateMsg; + vt_debug_print( + verbose, vrt_coll, + "migrateOut: (after remove) holder numElements={}\n", + elm_holder->numElements() + ); + + /* + * Invoke the virtual prelude migrate out function + */ + col_unique_ptr->preMigrateOut(); + + vt_debug_print( + verbose, vrt_coll, + "migrateOut: col_proxy={:x}, idx={}, dest={}: serializing collection elm\n", + col_proxy, print_index(idx), dest + ); - auto msg = makeMessage(proxy, this_node, dest, &typed_col_ref); + using MigrateMsgType = MigrateMsg; - theMsg()->sendMsg< - MigrateMsgType, MigrateHandlers::migrateInHandler - >(dest, msg); + auto msg = + makeMessage(proxy, this_node, dest, &typed_col_ref); - theLocMan()->getCollectionLM(col_proxy)->entityEmigrated(idx, dest); + theMsg()->sendMsg< + MigrateMsgType, MigrateHandlers::migrateInHandler + >(dest, msg); - /* - * Invoke the virtual epilog migrate out function - */ - col_unique_ptr->epiMigrateOut(); + theLocMan()->getCollectionLM(col_proxy)->entityEmigrated(idx, dest); - vt_debug_print( - verbose, vrt_coll, - "migrateOut: col_proxy={:x}, idx={}, dest={}: invoking destroy()\n", - col_proxy, print_index(idx), dest - ); + /* + * Invoke the virtual epilog migrate out function + */ + col_unique_ptr->epiMigrateOut(); - /* - * Invoke the virtual destroy function and then null std::unique_ptr, - * which should cause the destructor to fire - */ - col_unique_ptr->destroy(); - col_unique_ptr = nullptr; + vt_debug_print( + verbose, vrt_coll, + "migrateOut: col_proxy={:x}, idx={}, dest={}: invoking destroy()\n", + col_proxy, print_index(idx), dest + ); - auto const home_node = getMappedNode(col_proxy, idx); - elm_holder->applyListeners( - listener::ElementEventEnum::ElementMigratedOut, idx, home_node - ); + /* + * Invoke the virtual destroy function and then null std::unique_ptr, + * which should cause the destructor to fire + */ + col_unique_ptr->destroy(); + col_unique_ptr = nullptr; + + auto const home_node = getMappedNode(col_proxy, idx); + elm_holder->applyListeners( + listener::ElementEventEnum::ElementMigratedOut, idx, home_node + ); - return MigrateStatus::MigratedToRemote; - } else { - #if vt_check_enabled(runtime_checks) - vtAssert( - false, "Migration should only be called when to_node is != this_node" - ); - #else - // Do nothing - #endif - return MigrateStatus::NoMigrationNecessary; - } + return MigrateStatus::MigratedToRemote; + // } else { + // #if vt_check_enabled(runtime_checks) + // vtAssert( + // false, "Migration should only be called when to_node is != this_node" + // ); + // #else + // // Do nothing + // #endif + // return MigrateStatus::NoMigrationNecessary; + // } } template @@ -1906,10 +1910,11 @@ MigrateStatus CollectionManager::migrateIn( VirtualProxyType const& proxy, IndexT const& idx, NodeType const& from, VirtualPtrType vrt_elm_ptr ) { + auto const this_node = theContext()->getNode(); vt_debug_print( terse, vrt_coll, - "CollectionManager::migrateIn: proxy={:x}, idx={}, from={}, ptr={}\n", - proxy, print_index(idx), from, print_ptr(vrt_elm_ptr.get()) + "CollectionManager::migrateIn: proxy={:x}, idx={}, from={}, this_node={} ptr={}\n", + proxy, print_index(idx), from, this_node, print_ptr(vrt_elm_ptr.get()) ); auto vc_raw_ptr = vrt_elm_ptr.get(); @@ -1920,7 +1925,6 @@ MigrateStatus CollectionManager::migrateIn( vc_raw_ptr->preMigrateIn(); // Always update the element ID struct for LB statistic tracking - auto const& this_node = theContext()->getNode(); vrt_elm_ptr->elm_id_.curr_node = this_node; auto home_node = getMappedNode(proxy, idx); diff --git a/src/vt/vrt/collection/migrate/migrate_handlers.impl.h b/src/vt/vrt/collection/migrate/migrate_handlers.impl.h index bb68440d58..1cd2e38068 100644 --- a/src/vt/vrt/collection/migrate/migrate_handlers.impl.h +++ b/src/vt/vrt/collection/migrate/migrate_handlers.impl.h @@ -65,11 +65,12 @@ template auto const& col_proxy = full_proxy.getCollectionProxy(); auto const& elm_proxy = full_proxy.getElementProxy(); auto const& idx = elm_proxy.getIndex(); + auto const this_node = theContext()->getNode(); vt_debug_print( terse, vrt_coll, - "migrateInHandler: from_node={}, idx={}\n", - from_node, idx + "migrateInHandler: from_node={}, this_node={}, idx={}\n", + from_node, this_node, idx ); auto vc_elm_ptr = std::unique_ptr(msg->elm_);