#476 allow migrating to the same node

DARMA-tasking · Apr 21, 2022 · 64e207d · 64e207d
1 parent 25b4ece
commit 64e207d
Show file tree

Hide file tree

Showing 7 changed files with 120 additions and 111 deletions.
diff --git a/examples/collection/lb_iter.cc b/examples/collection/lb_iter.cc
@@ -66,6 +66,14 @@ struct IterCol : vt::Collection<IterCol, vt::Index1D> {
 
   template <typename SerializerT>
   void serialize(SerializerT& s) {
+    if (s.isSizing()) {
+      fmt::print("IterCol::serialize()::isSizing()\n");
+    } else if (s.isPacking()) {
+      fmt::print("IterCol::serialize()::isPacking()\n");
+    } else if (s.isUnpacking()) {
+      fmt::print("IterCol::serialize()::isUnpacking()\n");
+    }
+
     vt::Collection<IterCol, vt::Index1D>::serialize(s);
     s | data_2;
   }

diff --git a/src/vt/messaging/active.cc b/src/vt/messaging/active.cc
@@ -230,11 +230,6 @@ EventType ActiveMessenger::sendMsgBytesWithPut(
     );
   }
 
-  vtWarnIf(
-    !(dest != theContext()->getNode() || is_bcast),
-    fmt::format("Destination {} should != this node", dest)
-  );
-
   MsgSizeType new_msg_size = base.size();
 
   if (is_put && !is_put_packed) {

diff --git a/src/vt/vrt/collection/balance/baselb/baselb.cc b/src/vt/vrt/collection/balance/baselb/baselb.cc
@@ -142,11 +142,10 @@ std::shared_ptr<const balance::Reassignment> BaseLB::normalizeReassignments() {
     auto const new_node = std::get<1>(transfer);
     auto const current_node = obj_id.curr_node;
 
-    // self-migration
     if (current_node == new_node) {
-      // Filter out self-migrations entirely
-      continue;
-      // vtAbort("Not currently implemented -- self-migration");
+      vt_debug_print(
+        terse, lb, "BaseLB::normalizeReassignments(): self migration\n"
+       );
     }
 
     // the object lives here, so it's departing.
@@ -227,8 +226,8 @@ void BaseLB::notifyNewHostNodeOfObjectsArriving(
   }
 }
 
-void BaseLB::migrateObjectTo(ObjIDType const obj_id, NodeType const to) {
-  if (obj_id.curr_node != to) {
+void BaseLB::migrateObjectTo(ObjIDType const obj_id, NodeType const to, bool const allow_self_migration) {
+  if (obj_id.curr_node != to || allow_self_migration) {
     transfers_.push_back(TransferDestType{obj_id, to});
   }
 }

diff --git a/src/vt/vrt/collection/balance/baselb/baselb.h b/src/vt/vrt/collection/balance/baselb/baselb.h
@@ -126,7 +126,8 @@ struct BaseLB {
     TransferVecType const& transfers, MigrationCountCB migration_count_callback
   );
   void migrationDone();
-  void migrateObjectTo(ObjIDType const obj_id, NodeType const node);
+  // TODO (STRZ) - in the end don't use bool var
+  void migrateObjectTo(ObjIDType const obj_id, NodeType const node, bool const allow_self_migration = false);
   void transferSend(NodeType from, TransferVecType const& transfer);
   void transferMigrations(TransferMsg<TransferVecType>* msg);
   void finalize(CountMsg* msg);

diff --git a/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc b/src/vt/vrt/collection/balance/serdetestlb/serdetestlb.cc
@@ -81,7 +81,8 @@ void SerdeTestLB::runLB(TimeType) {
       obj, load, next_node, this_node
     );
     if (obj.isMigratable()) {
-      migrateObjectTo(obj, next_node);
+      constexpr bool allow_self_migration = true;
+      migrateObjectTo(obj, next_node, allow_self_migration);
     }
   }
 }

diff --git a/src/vt/vrt/collection/manager.impl.h b/src/vt/vrt/collection/manager.impl.h
@@ -1789,127 +1789,132 @@ template <typename ColT, typename IndexT>
 MigrateStatus CollectionManager::migrateOut(
   VirtualProxyType const& col_proxy, IndexT const& idx, NodeType const& dest
 ) {
- auto const& this_node = theContext()->getNode();
-
- vt_debug_print(
-   terse, vrt_coll,
-   "migrateOut: col_proxy={:x}, this_node={}, dest={}, "
-   "idx={}\n",
-   col_proxy, this_node, dest, print_index(idx)
- );
-
- if (this_node != dest) {
-   auto const& proxy = CollectionProxy<ColT, IndexT>(col_proxy).operator()(
-     idx
-   );
-   auto elm_holder = findElmHolder<IndexT>(col_proxy);
-   vtAssert(
-     elm_holder != nullptr, "Element must be registered here"
-   );
+  auto const& this_node = theContext()->getNode();
 
-   #if vt_check_enabled(runtime_checks)
-   {
-     bool const exists = elm_holder->exists(idx);
-     vtAssert(
-       exists, "Local element must exist here for migration to occur"
-     );
-   }
-   #endif
+  vt_debug_print(
+    terse, vrt_coll,
+    "migrateOut: col_proxy={:x}, this_node={}, dest={}, idx={}\n",
+    col_proxy, this_node, dest, print_index(idx)
+  );
 
-   bool const exists = elm_holder->exists(idx);
-   if (!exists) {
-     return MigrateStatus::ElementNotLocal;
-   }
+  // if (this_node != dest) {
+  vt_debug_print(
+    terse, vrt_coll, "migrating from {} to {}\n", this_node, dest
+  );
 
-   vt_debug_print(
-     verbose, vrt_coll,
-     "migrateOut: (before remove) holder numElements={}\n",
-     elm_holder->numElements()
-   );
+  auto const& proxy = CollectionProxy<ColT, IndexT>(col_proxy).operator()(
+    idx
+  );
+  auto elm_holder = findElmHolder<IndexT>(col_proxy);
+  vtAssert(
+    elm_holder != nullptr, "Element must be registered here"
+  );
 
-   if (elm_holder->numElements() == 1 and theConfig()->vt_lb_keep_last_elm) {
-     vt_debug_print(
-       normal, vrt_coll,
-       "migrateOut: do not migrate last element\n"
-     );
-     return MigrateStatus::ElementNotLocal;
-   }
+#if vt_check_enabled(runtime_checks)
+  {
+    bool const exists = elm_holder->exists(idx);
+    vtAssert(
+      exists, "Local element must exist here for migration to occur"
+    );
+  }
+#endif
 
-   auto col_unique_ptr = elm_holder->remove(idx);
-   auto& typed_col_ref = *static_cast<ColT*>(col_unique_ptr.get());
+  bool const exists = elm_holder->exists(idx);
+  if (!exists) {
+    return MigrateStatus::ElementNotLocal;
+  }
 
-   vt_debug_print(
-     verbose, vrt_coll,
-     "migrateOut: (after remove) holder numElements={}\n",
-     elm_holder->numElements()
-   );
+  vt_debug_print(
+    verbose, vrt_coll,
+    "migrateOut: (before remove) holder numElements={}\n",
+    elm_holder->numElements()
+  );
 
-   /*
-    * Invoke the virtual prelude migrate out function
-    */
-   col_unique_ptr->preMigrateOut();
+  if (elm_holder->numElements() == 1 and theConfig()->vt_lb_keep_last_elm) {
+    vt_debug_print(
+      normal, vrt_coll,
+      "migrateOut: do not migrate last element\n"
+    );
+    return MigrateStatus::ElementNotLocal;
+  }
 
-   vt_debug_print(
-     verbose, vrt_coll,
-     "migrateOut: col_proxy={:x}, idx={}, dest={}: serializing collection elm\n",
-     col_proxy, print_index(idx), dest
-   );
+  auto col_unique_ptr = elm_holder->remove(idx);
+  auto& typed_col_ref = *static_cast<ColT*>(col_unique_ptr.get());
 
-   using MigrateMsgType = MigrateMsg<ColT, IndexT>;
+  vt_debug_print(
+    verbose, vrt_coll,
+    "migrateOut: (after remove) holder numElements={}\n",
+    elm_holder->numElements()
+  );
+
+  /*
+  * Invoke the virtual prelude migrate out function
+  */
+  col_unique_ptr->preMigrateOut();
+
+  vt_debug_print(
+    verbose, vrt_coll,
+    "migrateOut: col_proxy={:x}, idx={}, dest={}: serializing collection elm\n",
+    col_proxy, print_index(idx), dest
+  );
 
-   auto msg = makeMessage<MigrateMsgType>(proxy, this_node, dest, &typed_col_ref);
+  using MigrateMsgType = MigrateMsg<ColT, IndexT>;
 
-   theMsg()->sendMsg<
-     MigrateMsgType, MigrateHandlers::migrateInHandler<ColT, IndexT>
-   >(dest, msg);
+  auto msg =
+    makeMessage<MigrateMsgType>(proxy, this_node, dest, &typed_col_ref);
 
-   theLocMan()->getCollectionLM<IndexT>(col_proxy)->entityEmigrated(idx, dest);
+  theMsg()->sendMsg<
+    MigrateMsgType, MigrateHandlers::migrateInHandler<ColT, IndexT>
+  >(dest, msg);
 
-   /*
-    * Invoke the virtual epilog migrate out function
-    */
-   col_unique_ptr->epiMigrateOut();
+  theLocMan()->getCollectionLM<IndexT>(col_proxy)->entityEmigrated(idx, dest);
 
-   vt_debug_print(
-     verbose, vrt_coll,
-     "migrateOut: col_proxy={:x}, idx={}, dest={}: invoking destroy()\n",
-     col_proxy, print_index(idx), dest
-   );
+  /*
+  * Invoke the virtual epilog migrate out function
+  */
+  col_unique_ptr->epiMigrateOut();
 
-   /*
-    * Invoke the virtual destroy function and then null std::unique_ptr<ColT>,
-    * which should cause the destructor to fire
-    */
-   col_unique_ptr->destroy();
-   col_unique_ptr = nullptr;
+  vt_debug_print(
+    verbose, vrt_coll,
+    "migrateOut: col_proxy={:x}, idx={}, dest={}: invoking destroy()\n",
+    col_proxy, print_index(idx), dest
+  );
 
-   auto const home_node = getMappedNode<IndexT>(col_proxy, idx);
-   elm_holder->applyListeners(
-     listener::ElementEventEnum::ElementMigratedOut, idx, home_node
-   );
+  /*
+  * Invoke the virtual destroy function and then null std::unique_ptr<ColT>,
+  * which should cause the destructor to fire
+  */
+  col_unique_ptr->destroy();
+  col_unique_ptr = nullptr;
+
+  auto const home_node = getMappedNode<IndexT>(col_proxy, idx);
+  elm_holder->applyListeners(
+    listener::ElementEventEnum::ElementMigratedOut, idx, home_node
+  );
 
-   return MigrateStatus::MigratedToRemote;
- } else {
-   #if vt_check_enabled(runtime_checks)
-     vtAssert(
-       false, "Migration should only be called when to_node is != this_node"
-     );
-   #else
-     // Do nothing
-   #endif
-   return MigrateStatus::NoMigrationNecessary;
- }
+  return MigrateStatus::MigratedToRemote;
+  // } else {
+  // #if vt_check_enabled(runtime_checks)
+  //   vtAssert(
+  //     false, "Migration should only be called when to_node is != this_node"
+  //   );
+  // #else
+  //     // Do nothing
+  // #endif
+  //   return MigrateStatus::NoMigrationNecessary;
+  // }
 }
 
 template <typename ColT, typename IndexT>
 MigrateStatus CollectionManager::migrateIn(
   VirtualProxyType const& proxy, IndexT const& idx, NodeType const& from,
   VirtualPtrType<IndexT> vrt_elm_ptr
 ) {
+  auto const this_node = theContext()->getNode();
   vt_debug_print(
     terse, vrt_coll,
-    "CollectionManager::migrateIn: proxy={:x}, idx={}, from={}, ptr={}\n",
-    proxy, print_index(idx), from, print_ptr(vrt_elm_ptr.get())
+    "CollectionManager::migrateIn: proxy={:x}, idx={}, from={}, this_node={} ptr={}\n",
+    proxy, print_index(idx), from, this_node, print_ptr(vrt_elm_ptr.get())
   );
 
   auto vc_raw_ptr = vrt_elm_ptr.get();
@@ -1920,7 +1925,6 @@ MigrateStatus CollectionManager::migrateIn(
   vc_raw_ptr->preMigrateIn();
 
   // Always update the element ID struct for LB statistic tracking
-  auto const& this_node = theContext()->getNode();
   vrt_elm_ptr->elm_id_.curr_node = this_node;
 
   auto home_node = getMappedNode<ColT>(proxy, idx);

diff --git a/src/vt/vrt/collection/migrate/migrate_handlers.impl.h b/src/vt/vrt/collection/migrate/migrate_handlers.impl.h
@@ -65,11 +65,12 @@ template <typename ColT, typename IndexT>
   auto const& col_proxy = full_proxy.getCollectionProxy();
   auto const& elm_proxy = full_proxy.getElementProxy();
   auto const& idx = elm_proxy.getIndex();
+  auto const this_node = theContext()->getNode();
 
   vt_debug_print(
     terse, vrt_coll,
-    "migrateInHandler: from_node={}, idx={}\n",
-    from_node, idx
+    "migrateInHandler: from_node={}, this_node={}, idx={}\n",
+    from_node, this_node, idx
   );
 
   auto vc_elm_ptr = std::unique_ptr<ColT>(msg->elm_);