Move the utility function to gather distributed vectors from tests/community to tests/utilities and update MG tests

cpp/include/cugraph/experimental/graph_functions.hpp

@@ -284,7 +284,7 @@ void unrenumber_int_vertices(raft::handle_t const& handle,
                              vertex_t const* renumber_map_labels,
                              vertex_t local_int_vertex_first,
                              vertex_t local_int_vertex_last,
-                             std::vector<vertex_t>& vertex_partition_lasts,
+                             std::vector<vertex_t> const& vertex_partition_lasts,
                              bool do_expensive_check = false);
 
 /**

cpp/include/cugraph/utilities/host_scalar_comm.cuh

@@ -15,6 +15,7 @@
  */
 #pragma once
 
+#include <cugraph/utilities/error.hpp>
 #include <cugraph/utilities/thrust_tuple_utils.cuh>
 
 #include <raft/handle.hpp>

cpp/src/experimental/renumber_utils.cu

@@ -214,7 +214,7 @@ void unrenumber_int_vertices(raft::handle_t const& handle,
                              vertex_t const* renumber_map_labels,
                              vertex_t local_int_vertex_first,
                              vertex_t local_int_vertex_last,
-                             std::vector<vertex_t>& vertex_partition_lasts,
+                             std::vector<vertex_t> const& vertex_partition_lasts,
                              bool do_expensive_check)
 {
   double constexpr load_factor = 0.7;
@@ -385,41 +385,45 @@ template void unrenumber_local_int_vertices<int64_t>(raft::handle_t const& handl
                                                      int64_t local_int_vertex_last,
                                                      bool do_expensive_check);
 
-template void unrenumber_int_vertices<int32_t, false>(raft::handle_t const& handle,
-                                                      int32_t* vertices,
-                                                      size_t num_vertices,
-                                                      int32_t const* renumber_map_labels,
-                                                      int32_t local_int_vertex_first,
-                                                      int32_t local_int_vertex_last,
-                                                      std::vector<int32_t>& vertex_partition_lasts,
-                                                      bool do_expensive_check);
+template void unrenumber_int_vertices<int32_t, false>(
+  raft::handle_t const& handle,
+  int32_t* vertices,
+  size_t num_vertices,
+  int32_t const* renumber_map_labels,
+  int32_t local_int_vertex_first,
+  int32_t local_int_vertex_last,
+  std::vector<int32_t> const& vertex_partition_lasts,
+  bool do_expensive_check);
 
-template void unrenumber_int_vertices<int32_t, true>(raft::handle_t const& handle,
-                                                     int32_t* vertices,
-                                                     size_t num_vertices,
-                                                     int32_t const* renumber_map_labels,
-                                                     int32_t local_int_vertex_first,
-                                                     int32_t local_int_vertex_last,
-                                                     std::vector<int32_t>& vertex_partition_lasts,
-                                                     bool do_expensive_check);
+template void unrenumber_int_vertices<int32_t, true>(
+  raft::handle_t const& handle,
+  int32_t* vertices,
+  size_t num_vertices,
+  int32_t const* renumber_map_labels,
+  int32_t local_int_vertex_first,
+  int32_t local_int_vertex_last,
+  std::vector<int32_t> const& vertex_partition_lasts,
+  bool do_expensive_check);
 
-template void unrenumber_int_vertices<int64_t, false>(raft::handle_t const& handle,
-                                                      int64_t* vertices,
-                                                      size_t num_vertices,
-                                                      int64_t const* renumber_map_labels,
-                                                      int64_t local_int_vertex_first,
-                                                      int64_t local_int_vertex_last,
-                                                      std::vector<int64_t>& vertex_partition_lasts,
-                                                      bool do_expensive_check);
+template void unrenumber_int_vertices<int64_t, false>(
+  raft::handle_t const& handle,
+  int64_t* vertices,
+  size_t num_vertices,
+  int64_t const* renumber_map_labels,
+  int64_t local_int_vertex_first,
+  int64_t local_int_vertex_last,
+  std::vector<int64_t> const& vertex_partition_lasts,
+  bool do_expensive_check);
 
-template void unrenumber_int_vertices<int64_t, true>(raft::handle_t const& handle,
-                                                     int64_t* vertices,
-                                                     size_t num_vertices,
-                                                     int64_t const* renumber_map_labels,
-                                                     int64_t local_int_vertex_first,
-                                                     int64_t local_int_vertex_last,
-                                                     std::vector<int64_t>& vertex_partition_lasts,
-                                                     bool do_expensive_check);
+template void unrenumber_int_vertices<int64_t, true>(
+  raft::handle_t const& handle,
+  int64_t* vertices,
+  size_t num_vertices,
+  int64_t const* renumber_map_labels,
+  int64_t local_int_vertex_first,
+  int64_t local_int_vertex_last,
+  std::vector<int64_t> const& vertex_partition_lasts,
+  bool do_expensive_check);
 
 }  // namespace experimental
 }  // namespace cugraph

cpp/tests/CMakeLists.txt

@@ -58,6 +58,39 @@ target_link_libraries(cugraphtestutil cugraph)
 set_target_properties(cugraphtestutil PROPERTIES
         CUDA_ARCHITECTURES OFF)
 
+add_library(cugraphmgtestutil STATIC
+            "${CMAKE_CURRENT_SOURCE_DIR}/utilities/device_comm_wrapper.cu")
+
+set_property(TARGET cugraphmgtestutil PROPERTY POSITION_INDEPENDENT_CODE ON)
+
+target_include_directories(cugraphmgtestutil
+    PRIVATE
+    "${CUB_INCLUDE_DIR}"
+    "${THRUST_INCLUDE_DIR}"
+    "${CUCO_INCLUDE_DIR}"
+    "${LIBCUDACXX_INCLUDE_DIR}"
+    "${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}"
+    "${RMM_INCLUDE}"
+    "${NCCL_INCLUDE_DIRS}"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../../thirdparty/mmio"
+    "${CMAKE_CURRENT_SOURCE_DIR}/../include"
+    "${CMAKE_CURRENT_SOURCE_DIR}"
+    "${RAFT_DIR}/cpp/include"
+)
+
+target_link_libraries(cugraphmgtestutil cugraph)
+
+# CUDA_ARCHITECTURES=OFF implies cmake will not pass arch flags to the
+# compiler. CUDA_ARCHITECTURES must be set to a non-empty value to prevent
+# cmake warnings about policy CMP0104. With this setting, arch flags must be
+# manually set! ("evaluate_gpu_archs(GPU_ARCHS)" is the current mechanism
+# used in cpp/CMakeLists.txt for setting arch options).
+# Run "cmake --help-policy CMP0104" for policy details.
+# NOTE: the CUDA_ARCHITECTURES=OFF setting may be removed after migrating to
+# the findcudatoolkit features in cmake 3.17+
+set_target_properties(cugraphmgtestutil PROPERTIES
+        CUDA_ARCHITECTURES OFF)
+
 ###################################################################################################
 # - compiler function -----------------------------------------------------------------------------
 
@@ -194,6 +227,7 @@ function(ConfigureTestMG CMAKE_TEST_NAME CMAKE_TEST_SRC)
 
     target_link_libraries(${CMAKE_TEST_NAME}
         PRIVATE
+        cugraphmgtestutil
         cugraphtestutil
         cugraph
         GTest::GTest

cpp/tests/community/mg_louvain_helper.cu

@@ -31,36 +31,6 @@
 namespace cugraph {
 namespace test {
 
-template <typename T>
-rmm::device_uvector<T> gather_distributed_vector(raft::handle_t const &handle,
-                                                 T const *d_input,
-                                                 size_t size)
-{
-  auto rx_sizes =
-    cugraph::experimental::host_scalar_gather(handle.get_comms(), size, 0, handle.get_stream());
-  std::vector<size_t> rx_displs(static_cast<size_t>(handle.get_comms().get_rank()) == 0
-                                  ? handle.get_comms().get_size()
-                                  : int{0},
-                                size_t{0});
-  if (static_cast<size_t>(handle.get_comms().get_rank()) == 0) {
-    std::partial_sum(rx_sizes.begin(), rx_sizes.end() - 1, rx_displs.begin() + 1);
-  }
-
-  auto total_size = thrust::reduce(thrust::host, rx_sizes.begin(), rx_sizes.end());
-  rmm::device_uvector<T> gathered_v(total_size, handle.get_stream());
-
-  cugraph::experimental::device_gatherv(handle.get_comms(),
-                                        d_input,
-                                        gathered_v.data(),
-                                        size,
-                                        rx_sizes,
-                                        rx_displs,
-                                        0,
-                                        handle.get_stream());
-
-  return gathered_v;
-}
-
 template <typename vertex_t>
 bool compare_renumbered_vectors(raft::handle_t const &handle,
                                 rmm::device_uvector<vertex_t> const &v1,
@@ -336,10 +306,6 @@ template void single_gpu_renumber_edgelist_given_number_map(
   rmm::device_uvector<int> &d_edgelist_cols,
   rmm::device_uvector<int> &d_renumber_map_gathered_v);
 
-template rmm::device_uvector<int> gather_distributed_vector(raft::handle_t const &handle,
-                                                            int const *d_input,
-                                                            size_t size);
-
 template bool compare_renumbered_vectors(raft::handle_t const &handle,
                                          rmm::device_uvector<int> const &v1,
                                          rmm::device_uvector<int> const &v2);

cpp/tests/community/mg_louvain_helper.hpp

@@ -24,11 +24,6 @@
 namespace cugraph {
 namespace test {
 
-template <typename T>
-rmm::device_uvector<T> gather_distributed_vector(raft::handle_t const &handle,
-                                                 T const *d_input,
-                                                 size_t size);
-
 template <typename vertex_t>
 bool compare_renumbered_vectors(raft::handle_t const &handle,
                                 rmm::device_uvector<vertex_t> const &v1,

cpp/tests/community/mg_louvain_test.cpp

@@ -17,6 +17,7 @@
 #include "mg_louvain_helper.hpp"
 
 #include <utilities/base_fixture.hpp>
+#include <utilities/device_comm_wrapper.hpp>
 #include <utilities/test_utilities.hpp>
 
 #include <cugraph/algorithms.hpp>
@@ -144,7 +145,7 @@ class Louvain_MG_Testfixture : public ::testing::TestWithParam<Louvain_Usecase>
       thrust::make_counting_iterator<size_t>(dendrogram.num_levels()),
       [&dendrogram, &sg_graph, &d_clustering_v, &sg_modularity, &handle, resolution, rank](
         size_t i) {
-        auto d_dendrogram_gathered_v = cugraph::test::gather_distributed_vector(
+        auto d_dendrogram_gathered_v = cugraph::test::device_gatherv(
           handle, dendrogram.get_level_ptr_nocheck(i), dendrogram.get_level_size_nocheck(i));
 
         if (rank == 0) {
@@ -207,7 +208,7 @@ class Louvain_MG_Testfixture : public ::testing::TestWithParam<Louvain_Usecase>
 
     SCOPED_TRACE("compare modularity input: " + param.graph_file_full_path);
 
-    auto d_renumber_map_gathered_v = cugraph::test::gather_distributed_vector(
+    auto d_renumber_map_gathered_v = cugraph::test::device_gatherv(
       handle, d_renumber_map_labels.data(), d_renumber_map_labels.size());
 
     compare_sg_results<vertex_t, edge_t, weight_t>(handle,

cpp/tests/experimental/bfs_test.cpp

@@ -212,12 +212,15 @@ class Tests_BFS : public ::testing::TestWithParam<std::tuple<BFS_Usecase, input_
                                                              graph_view.get_number_of_vertices(),
                                                              true);
 
-        auto d_unrenumbered_distances = cugraph::test::sort_by_key(
+        rmm::device_uvector<vertex_t> d_unrenumbered_distances(size_t{0}, handle.get_stream());
+        std::tie(std::ignore, d_unrenumbered_distances) = cugraph::test::sort_by_key(
           handle, d_renumber_map_labels.data(), d_distances.data(), d_renumber_map_labels.size());
-        auto d_unrenumbered_predecessors = cugraph::test::sort_by_key(handle,
-                                                                      d_renumber_map_labels.data(),
-                                                                      d_predecessors.data(),
-                                                                      d_renumber_map_labels.size());
+        rmm::device_uvector<vertex_t> d_unrenumbered_predecessors(size_t{0}, handle.get_stream());
+        std::tie(std::ignore, d_unrenumbered_predecessors) =
+          cugraph::test::sort_by_key(handle,
+                                     d_renumber_map_labels.data(),
+                                     d_predecessors.data(),
+                                     d_renumber_map_labels.size());
         raft::update_host(h_cugraph_distances.data(),
                           d_unrenumbered_distances.data(),
                           d_unrenumbered_distances.size(),

cpp/tests/experimental/katz_centrality_test.cpp

@@ -225,7 +225,9 @@ class Tests_KatzCentrality
 
       std::vector<result_t> h_cugraph_katz_centralities(graph_view.get_number_of_vertices());
       if (renumber) {
-        auto d_unrenumbered_katz_centralities =
+        rmm::device_uvector<result_t> d_unrenumbered_katz_centralities(size_t{0},
+                                                                       handle.get_stream());
+        std::tie(std::ignore, d_unrenumbered_katz_centralities) =
           cugraph::test::sort_by_key(handle,
                                      d_renumber_map_labels.data(),
                                      d_katz_centralities.data(),