From 29f6a8bf3f8ec75c41cee9a8cc2e0535668bf541 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Tue, 17 May 2022 13:17:45 +0000 Subject: [PATCH 01/14] sg and mg bfs plc migration work --- python/cufile.log | 46 +++++++ python/cugraph/cugraph/dask/traversal/bfs.py | 68 +++++++++-- python/cugraph/cugraph/traversal/bfs.pxd | 35 ------ python/cugraph/cugraph/traversal/bfs.py | 55 ++++++++- .../cugraph/cugraph/traversal/bfs_wrapper.pyx | 113 ------------------ python/pylibcugraph/pylibcugraph/__init__.py | 2 + python/pylibcugraph/pylibcugraph/bfs.pyx | 105 ++++++++++++++++ .../pylibcugraph/experimental/__init__.py | 3 + 8 files changed, 266 insertions(+), 161 deletions(-) create mode 100644 python/cufile.log delete mode 100644 python/cugraph/cugraph/traversal/bfs.pxd delete mode 100644 python/cugraph/cugraph/traversal/bfs_wrapper.pyx create mode 100644 python/pylibcugraph/pylibcugraph/bfs.pyx diff --git a/python/cufile.log b/python/cufile.log new file mode 100644 index 00000000000..8f26cc2baa6 --- /dev/null +++ b/python/cufile.log @@ -0,0 +1,46 @@ + 17-05-2022 02:01:24:56 [pid=31487 tid=31487] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 02:03:46:442 [pid=32829 tid=32829] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 02:04:27:698 [pid=33451 tid=33451] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 02:06:59:846 [pid=35194 tid=35194] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 02:10:24:331 [pid=37240 tid=37240] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 02:11:15:171 [pid=37788 tid=37788] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 02:12:36:46 [pid=38616 tid=38616] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 02:13:09:49 [pid=39020 tid=39020] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 02:13:50:923 [pid=39637 tid=39637] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 02:20:25:531 [pid=43557 tid=43557] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 03:03:01:752 [pid=66356 tid=66356] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 03:03:01:927 [pid=66536 tid=66950] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 03:03:01:928 [pid=66524 tid=66952] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 03:03:01:928 [pid=66528 tid=66956] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 03:03:01:929 [pid=66518 tid=66953] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 03:03:01:929 [pid=66533 tid=66951] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 03:03:01:929 [pid=66540 tid=66954] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 03:03:01:929 [pid=66530 tid=66957] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 03:03:01:930 [pid=66512 tid=66955] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 03:09:45:446 [pid=70651 tid=70651] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 03:09:45:596 [pid=70810 tid=71256] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 03:09:45:603 [pid=70820 tid=71257] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 03:09:45:604 [pid=70802 tid=71258] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 03:09:45:604 [pid=70806 tid=71261] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 03:09:45:604 [pid=70825 tid=71262] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 03:09:45:604 [pid=70814 tid=71263] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 03:09:45:605 [pid=70822 tid=71259] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 03:09:45:605 [pid=70817 tid=71260] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 13:14:30:240 [pid=13992 tid=13992] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 13:14:30:394 [pid=14162 tid=14587] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 13:14:30:397 [pid=14151 tid=14588] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 13:14:30:397 [pid=14166 tid=14589] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 13:14:30:399 [pid=14152 tid=14592] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 13:14:30:399 [pid=14143 tid=14591] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 13:14:30:400 [pid=14148 tid=14590] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 13:14:30:401 [pid=14158 tid=14593] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 13:14:30:401 [pid=14160 tid=14594] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 13:15:12:815 [pid=14803 tid=14803] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 13:15:12:968 [pid=14954 tid=15397] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 13:15:12:969 [pid=14947 tid=15403] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 13:15:12:970 [pid=14961 tid=15399] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 13:15:12:970 [pid=14964 tid=15400] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 13:15:12:970 [pid=14967 tid=15398] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 13:15:12:970 [pid=14952 tid=15401] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 13:15:12:971 [pid=14959 tid=15402] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 13:15:12:971 [pid=14970 tid=15404] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded diff --git a/python/cugraph/cugraph/dask/traversal/bfs.py b/python/cugraph/cugraph/dask/traversal/bfs.py index ab9dbde1091..f934319cd64 100644 --- a/python/cugraph/cugraph/dask/traversal/bfs.py +++ b/python/cugraph/cugraph/dask/traversal/bfs.py @@ -13,12 +13,16 @@ # limitations under the License. # -from collections.abc import Iterable +from pylibcugraph.experimental import (MGGraph, + ResourceHandle, + GraphProperties, + bfs as pylibcugraph_bfs, + ) +from collections.abc import Iterable from dask.distributed import wait, default_client from cugraph.dask.common.input_utils import (get_distributed_data, get_vertex_partition_offsets) -from cugraph.dask.traversal import mg_bfs_wrapper as mg_bfs import cugraph.dask.comms.comms as Comms import cudf import dask_cudf @@ -53,6 +57,55 @@ def call_bfs(sID, depth_limit, return_distances) +def _call_plc_mg_bfs( + sID, + data, + sources, + depth_limit, + src_col_name, + dst_col_name, + num_edges, + direction_optimizing=False, + do_expensive_check=False, + return_predecessors=True): + comms_handle = Comms.get_handle(sID) + resource_handle = ResourceHandle(comms_handle.getHandle()) + + srcs = data[0][src_col_name] + dsts = data[0][dst_col_name] + weights = data[0]['value'] \ + if 'value' in data[0].columns \ + else cudf.Series((srcs + 1) / (srcs + 1), dtype='float32') + + + mg = MGGraph( + resource_handle = resource_handle, + graph_properties = GraphProperties(is_symmetric=False, is_multigraph=False), + src_array = srcs, + dst_array = dsts, + weights = weights, + store_transposed = False, + num_edges = num_edges, + do_expensive_check = do_expensive_check + ) + + distances, predecessors, vertices = \ + pylibcugraph_bfs( + resource_handle, + mg, + sources, + direction_optimizing, + depth_limit if depth_limit is not None else -1, + return_predecessors, + do_expensive_check + ) + + return cudf.DataFrame({ + 'distance': cudf.Series(distances), + 'vertex': cudf.Series(vertices), + 'predecessor': cudf.Series(predecessors), + }) + def bfs(input_graph, start, @@ -176,17 +229,16 @@ def df_merge(df, tmp_df, tmp_col_names): dst_col_name = input_graph.destination_columns result = [client.submit( - call_bfs, + _call_plc_mg_bfs, Comms.get_session_id(), wf[1], + start[idx], + depth_limit, src_col_name, dst_col_name, - num_verts, num_edges, - vertex_partition_offsets, - input_graph.aggregate_segment_offsets, - start[idx], - depth_limit, + False, + False, return_distances, workers=[wf[0]]) for idx, wf in enumerate(data.worker_to_parts.items())] diff --git a/python/cugraph/cugraph/traversal/bfs.pxd b/python/cugraph/cugraph/traversal/bfs.pxd deleted file mode 100644 index c4dbe6dddbc..00000000000 --- a/python/cugraph/cugraph/traversal/bfs.pxd +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) 2019-2021, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -from cugraph.structure.graph_utilities cimport * -from libcpp cimport bool - -cdef extern from "limits.h": - cdef int INT_MAX - -cdef extern from "cugraph/utilities/cython.hpp" namespace "cugraph::cython": - cdef void call_bfs[vertex_t, weight_t]( - const handle_t &handle, - const graph_container_t &g, - vertex_t *identifiers, - vertex_t *distances, - vertex_t *predecessors, - vertex_t depth_limit, - const vertex_t *start_vertex, - size_t n_sources, - bool direction_optimizing) except + diff --git a/python/cugraph/cugraph/traversal/bfs.py b/python/cugraph/cugraph/traversal/bfs.py index 50d943e1551..a8a312752a2 100644 --- a/python/cugraph/cugraph/traversal/bfs.py +++ b/python/cugraph/cugraph/traversal/bfs.py @@ -13,7 +13,12 @@ import cudf -from cugraph.traversal import bfs_wrapper +from pylibcugraph.experimental import (ResourceHandle, + GraphProperties, + SGGraph, + bfs as pylibcugraph_bfs, + ) + from cugraph.structure.graph_classes import Graph, DiGraph from cugraph.utilities import (ensure_cugraph_obj, is_matrix_type, @@ -76,13 +81,52 @@ def _convert_df_to_output_type(df, input_type): else: raise TypeError(f"input type {input_type} is not a supported type.") +def _call_plc_bfs(G, sources, depth_limit, do_expensive_check=False, + direction_optimizing=False, return_predecessors=True): + handle = ResourceHandle() + + srcs = G.edgelist.edgelist_df['src'] + dsts = G.edgelist.edgelist_df['dst'] + weights = G.edgelist.edgelist_df['weights'] \ + if 'weights' in G.edgelist.edgelist_df \ + else cudf.Series((srcs + 1) / (srcs + 1), dtype='float32') + + sg = SGGraph( + resource_handle = handle, + graph_properties = GraphProperties(is_multigraph=G.is_multigraph()), + src_array = srcs, + dst_array = dsts, + weight_array = weights, + store_transposed=False, + renumber=False, + do_expensive_check=do_expensive_check + ) + + distances, predecessors, vertices = \ + pylibcugraph_bfs( + handle, + sg, + sources, + direction_optimizing, + depth_limit if depth_limit is not None else -1, + return_predecessors, + do_expensive_check + ) + + return cudf.DataFrame({ + 'distance': cudf.Series(distances), + 'vertex': cudf.Series(vertices), + 'predecessor': cudf.Series(predecessors), + }) + + def bfs(G, start=None, depth_limit=None, i_start=None, directed=None, - return_predecessors=None): + return_predecessors=True): """ Find the distances and predecessors for a breadth first traversal of a graph. @@ -112,8 +156,9 @@ def bfs(G, If True, then convert the input matrix to a directed cugraph.Graph, otherwise an undirected cugraph.Graph object will be used. - return_predecessors : - + return_predecessors : bool, optional (default=True) + Whether to return the predecessors for each vertex (returns -1 + for each vertex otherwise) Returns ------- @@ -177,7 +222,7 @@ def bfs(G, else: start = cudf.Series(start) - df = bfs_wrapper.bfs(G, start, depth_limit) + df = _call_plc_bfs(G, start, depth_limit, return_predecessors=return_predecessors) if G.renumbered: df = G.unrenumber(df, "vertex") df = G.unrenumber(df, "predecessor") diff --git a/python/cugraph/cugraph/traversal/bfs_wrapper.pyx b/python/cugraph/cugraph/traversal/bfs_wrapper.pyx deleted file mode 100644 index a67cedc50cb..00000000000 --- a/python/cugraph/cugraph/traversal/bfs_wrapper.pyx +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright (c) 2019-2021, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -cimport cugraph.traversal.bfs as c_bfs -from cugraph.structure.graph_utilities cimport * -from cugraph.structure import graph_primtypes_wrapper -from libcpp cimport bool -from libc.stdint cimport uintptr_t -import cudf -import numpy as np - -def bfs(input_graph, start, depth_limit, direction_optimizing=False): - """ - Call bfs - """ - # Step 1: Declare the different varibales - cdef graph_container_t graph_container - - numberTypeMap = {np.dtype("int32") : numberTypeEnum.int32Type, - np.dtype("int64") : numberTypeEnum.int64Type, - np.dtype("float32") : numberTypeEnum.floatType, - np.dtype("double") : numberTypeEnum.doubleType} - - weight_t = np.dtype("float32") - [src, dst] = graph_primtypes_wrapper.datatype_cast([input_graph.edgelist.edgelist_df['src'], input_graph.edgelist.edgelist_df['dst']], [np.int32]) - weights = None - - # Pointers for SSSP / BFS - cdef uintptr_t c_identifier_ptr = NULL # Pointer to the DataFrame 'vertex' Series - cdef uintptr_t c_distance_ptr = NULL # Pointer to the DataFrame 'distance' Series - cdef uintptr_t c_predecessor_ptr = NULL # Pointer to the DataFrame 'predecessor' Series - if depth_limit is None: - depth_limit = c_bfs.INT_MAX - - # Step 2: Verifiy input_graph has the expected format - - cdef unique_ptr[handle_t] handle_ptr - handle_ptr.reset(new handle_t()) - handle_ = handle_ptr.get(); - - # Step 3: Setup number of vertices and edges - num_verts = input_graph.number_of_vertices() - num_edges = input_graph.number_of_edges(directed_edges=True) - - # Step 4: Extract COO - cdef uintptr_t c_src_vertices = src.__cuda_array_interface__['data'][0] - cdef uintptr_t c_dst_vertices = dst.__cuda_array_interface__['data'][0] - cdef uintptr_t c_edge_weights = NULL - - # Step 5: Check if source index is valid - # FIXME: Updates to multi-seed BFS support disabled this check. Re-enable ASAP. - #if not 0 <= start < num_verts: - # raise ValueError("Starting vertex should be between 0 to number of vertices") - - # Step 6: Generate the cudf.DataFrame result - # Current implementation expects int (signed 32-bit) for distance - df = cudf.DataFrame() - df['vertex'] = cudf.Series(np.arange(num_verts), dtype=np.int32) - df['distance'] = cudf.Series(np.zeros(num_verts, dtype=np.int32)) - df['predecessor'] = cudf.Series(np.zeros(num_verts, dtype=np.int32)) - - # Step 7: Associate to cudf Series - c_identifier_ptr = df['vertex'].__cuda_array_interface__['data'][0] - c_distance_ptr = df['distance'].__cuda_array_interface__['data'][0] - c_predecessor_ptr = df['predecessor'].__cuda_array_interface__['data'][0] - cdef uintptr_t c_start_ptr = start.__cuda_array_interface__['data'][0] - - is_symmetric = not input_graph.is_directed() - - # Step 8: Proceed to BFS - populate_graph_container(graph_container, - handle_[0], - c_src_vertices, c_dst_vertices, c_edge_weights, - NULL, - NULL, - 0, - ((numberTypeEnum.int32Type)), - ((numberTypeEnum.int32Type)), - ((numberTypeMap[weight_t])), - num_edges, - num_verts, num_edges, - False, - is_symmetric, - False, - False) - - # Different pathing wether shortest_path_counting is required or not - c_bfs.call_bfs[int, float](handle_ptr.get()[0], - graph_container, - c_identifier_ptr, - c_distance_ptr, - c_predecessor_ptr, - depth_limit, - c_start_ptr, - len(start), - direction_optimizing) - - return df diff --git a/python/pylibcugraph/pylibcugraph/__init__.py b/python/pylibcugraph/pylibcugraph/__init__.py index 96cee8e43c1..2406cfbdf5b 100644 --- a/python/pylibcugraph/pylibcugraph/__init__.py +++ b/python/pylibcugraph/pylibcugraph/__init__.py @@ -34,3 +34,5 @@ from pylibcugraph.hits import hits from pylibcugraph.node2vec import node2vec + +from pylibcugraph.bfs import bfs \ No newline at end of file diff --git a/python/pylibcugraph/pylibcugraph/bfs.pyx b/python/pylibcugraph/pylibcugraph/bfs.pyx new file mode 100644 index 00000000000..7eabff95749 --- /dev/null +++ b/python/pylibcugraph/pylibcugraph/bfs.pyx @@ -0,0 +1,105 @@ +from libc.stdint cimport uintptr_t +from libc.stdint cimport int64_t +from libc.limits cimport INT_MAX + +from pylibcugraph.resource_handle cimport ResourceHandle +from pylibcugraph._cugraph_c.algorithms cimport ( + cugraph_bfs, + cugraph_paths_result_t, + cugraph_paths_result_get_vertices, + cugraph_paths_result_get_predecessors, + cugraph_paths_result_get_distances, + cugraph_paths_result_free, +) +from pylibcugraph._cugraph_c.array cimport ( + cugraph_type_erased_device_array_view_t, + cugraph_type_erased_device_array_view_create, + cugraph_type_erased_device_array_view_free, +) +from pylibcugraph._cugraph_c.resource_handle cimport ( + bool_t, + data_type_id_t, + cugraph_resource_handle_t, +) +from pylibcugraph._cugraph_c.error cimport ( + cugraph_error_code_t, + cugraph_error_t, +) +from pylibcugraph.utils cimport ( + assert_success, + copy_to_cupy_array, + assert_CAI_type, + get_c_type_from_numpy_type, +) +from pylibcugraph._cugraph_c.graph cimport ( + cugraph_graph_t, +) +from pylibcugraph.graphs cimport ( + _GPUGraph, +) + +def bfs(ResourceHandle handle, _GPUGraph graph, + sources, bool_t direction_optimizing, int64_t depth_limit, + bool_t compute_predecessors, bool_t do_expensive_check): + try: + import cupy + except ModuleNotFoundError: + raise RuntimeError("bfs requires the cupy package, which could not " + "be imported") + assert_CAI_type(sources, "sources") + + if depth_limit < 0: + depth_limit = INT_MAX + + cdef cugraph_resource_handle_t* c_resource_handle_ptr = \ + handle.c_resource_handle_ptr + cdef cugraph_graph_t* c_graph_ptr = graph.c_graph_ptr + + cdef cugraph_error_code_t error_code + cdef cugraph_error_t* error_ptr + + cdef uintptr_t cai_sources_ptr = \ + sources.__cuda_array_interface__["data"][0] + cdef cugraph_type_erased_device_array_view_t* sources_view_ptr = \ + cugraph_type_erased_device_array_view_create( + cai_sources_ptr, + len(sources), + get_c_type_from_numpy_type(sources.dtype)) + + cdef cugraph_paths_result_t* result_ptr + + error_code = cugraph_bfs( + c_resource_handle_ptr, + c_graph_ptr, + sources_view_ptr, + direction_optimizing, + depth_limit, + compute_predecessors, + do_expensive_check, + &result_ptr, + &error_ptr + ) + assert_success(error_code, error_ptr, "cugraph_bfs") + + # deallocate the no-longer needed sources array + cugraph_type_erased_device_array_view_free(sources_view_ptr) + + # Extract individual device array pointers from result + cdef cugraph_type_erased_device_array_view_t* distances_ptr = \ + cugraph_paths_result_get_distances(result_ptr) + + cdef cugraph_type_erased_device_array_view_t* predecessors_ptr = \ + cugraph_paths_result_get_predecessors(result_ptr) + + cdef cugraph_type_erased_device_array_view_t* vertices_ptr = \ + cugraph_paths_result_get_vertices(result_ptr) + + # copy to cupy arrays + cupy_distances = copy_to_cupy_array(c_resource_handle_ptr, distances_ptr) + cupy_predecessors = copy_to_cupy_array(c_resource_handle_ptr, predecessors_ptr) + cupy_vertices = copy_to_cupy_array(c_resource_handle_ptr, vertices_ptr) + + # deallocate the no-longer needed result struct + cugraph_paths_result_free(result_ptr) + + return (cupy_distances, cupy_predecessors, cupy_vertices) \ No newline at end of file diff --git a/python/pylibcugraph/pylibcugraph/experimental/__init__.py b/python/pylibcugraph/pylibcugraph/experimental/__init__.py index 2587b2457a1..c4b53e0229e 100644 --- a/python/pylibcugraph/pylibcugraph/experimental/__init__.py +++ b/python/pylibcugraph/pylibcugraph/experimental/__init__.py @@ -66,6 +66,9 @@ from pylibcugraph.node2vec import node2vec node2vec = promoted_experimental_warning_wrapper(node2vec) +from pylibcugraph.bfs import bfs +bfs = promoted_experimental_warning_wrapper(bfs) + from pylibcugraph.uniform_neighborhood_sampling import EXPERIMENTAL__uniform_neighborhood_sampling uniform_neighborhood_sampling = experimental_warning_wrapper(EXPERIMENTAL__uniform_neighborhood_sampling) From 3c00af1d99c5a94700dbbb3b1724204fabc54fd5 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Tue, 17 May 2022 13:21:45 +0000 Subject: [PATCH 02/14] remove cufile --- python/cufile.log | 46 ---------------------------------------------- 1 file changed, 46 deletions(-) delete mode 100644 python/cufile.log diff --git a/python/cufile.log b/python/cufile.log deleted file mode 100644 index 8f26cc2baa6..00000000000 --- a/python/cufile.log +++ /dev/null @@ -1,46 +0,0 @@ - 17-05-2022 02:01:24:56 [pid=31487 tid=31487] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 02:03:46:442 [pid=32829 tid=32829] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 02:04:27:698 [pid=33451 tid=33451] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 02:06:59:846 [pid=35194 tid=35194] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 02:10:24:331 [pid=37240 tid=37240] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 02:11:15:171 [pid=37788 tid=37788] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 02:12:36:46 [pid=38616 tid=38616] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 02:13:09:49 [pid=39020 tid=39020] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 02:13:50:923 [pid=39637 tid=39637] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 02:20:25:531 [pid=43557 tid=43557] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 03:03:01:752 [pid=66356 tid=66356] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 03:03:01:927 [pid=66536 tid=66950] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 03:03:01:928 [pid=66524 tid=66952] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 03:03:01:928 [pid=66528 tid=66956] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 03:03:01:929 [pid=66518 tid=66953] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 03:03:01:929 [pid=66533 tid=66951] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 03:03:01:929 [pid=66540 tid=66954] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 03:03:01:929 [pid=66530 tid=66957] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 03:03:01:930 [pid=66512 tid=66955] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 03:09:45:446 [pid=70651 tid=70651] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 03:09:45:596 [pid=70810 tid=71256] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 03:09:45:603 [pid=70820 tid=71257] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 03:09:45:604 [pid=70802 tid=71258] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 03:09:45:604 [pid=70806 tid=71261] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 03:09:45:604 [pid=70825 tid=71262] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 03:09:45:604 [pid=70814 tid=71263] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 03:09:45:605 [pid=70822 tid=71259] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 03:09:45:605 [pid=70817 tid=71260] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 13:14:30:240 [pid=13992 tid=13992] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 13:14:30:394 [pid=14162 tid=14587] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 13:14:30:397 [pid=14151 tid=14588] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 13:14:30:397 [pid=14166 tid=14589] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 13:14:30:399 [pid=14152 tid=14592] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 13:14:30:399 [pid=14143 tid=14591] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 13:14:30:400 [pid=14148 tid=14590] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 13:14:30:401 [pid=14158 tid=14593] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 13:14:30:401 [pid=14160 tid=14594] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 13:15:12:815 [pid=14803 tid=14803] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 13:15:12:968 [pid=14954 tid=15397] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 13:15:12:969 [pid=14947 tid=15403] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 13:15:12:970 [pid=14961 tid=15399] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 13:15:12:970 [pid=14964 tid=15400] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 13:15:12:970 [pid=14967 tid=15398] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 13:15:12:970 [pid=14952 tid=15401] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 13:15:12:971 [pid=14959 tid=15402] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 13:15:12:971 [pid=14970 tid=15404] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded From 42c9e16bc64180a639a60652afc4e9934cc4efee Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Wed, 18 May 2022 20:19:55 +0000 Subject: [PATCH 03/14] add bfs mg c api test, small fixes --- cpp/tests/CMakeLists.txt | 1 + cpp/tests/c_api/mg_bfs_test.c | 180 +++++++ python/.gitignore | 1 + python/cufile.log | 152 ++++++ python/cugraph/cugraph/dask/traversal/bfs.py | 187 +++---- .../cugraph/traversal/sssp_wrapper.pyx | 1 - python/pylibcugraph/pylibcugraph/bfs.pyx | 8 +- testing/cufile.log | 480 ++++++++++++++++++ testing/dask-worker-space/global.lock | 0 testing/dask-worker-space/purge.lock | 0 .../dask-worker-space/worker-i66t8r65.dirlock | 0 .../dask-worker-space/worker-ifem89p4.dirlock | 0 .../dask-worker-space/worker-jvu0j__k.dirlock | 0 .../dask-worker-space/worker-l968y9vr.dirlock | 0 .../dask-worker-space/worker-lr81p2a2.dirlock | 0 .../dask-worker-space/worker-wjcu9c0j.dirlock | 0 .../dask-worker-space/worker-xtysqw6a.dirlock | 0 .../dask-worker-space/worker-ykczxdz3.dirlock | 0 testing/testing1.ipynb | 125 +++++ testing/testing1.py | 37 ++ 20 files changed, 1059 insertions(+), 113 deletions(-) create mode 100644 cpp/tests/c_api/mg_bfs_test.c create mode 100644 python/cufile.log create mode 100644 testing/cufile.log create mode 100644 testing/dask-worker-space/global.lock create mode 100644 testing/dask-worker-space/purge.lock create mode 100644 testing/dask-worker-space/worker-i66t8r65.dirlock create mode 100644 testing/dask-worker-space/worker-ifem89p4.dirlock create mode 100644 testing/dask-worker-space/worker-jvu0j__k.dirlock create mode 100644 testing/dask-worker-space/worker-l968y9vr.dirlock create mode 100644 testing/dask-worker-space/worker-lr81p2a2.dirlock create mode 100644 testing/dask-worker-space/worker-wjcu9c0j.dirlock create mode 100644 testing/dask-worker-space/worker-xtysqw6a.dirlock create mode 100644 testing/dask-worker-space/worker-ykczxdz3.dirlock create mode 100644 testing/testing1.ipynb create mode 100644 testing/testing1.py diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index cab8966c0f6..691710faf2f 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -643,6 +643,7 @@ if(BUILD_CUGRAPH_MG_TESTS) ConfigureCTestMG(MG_CAPI_HITS c_api/mg_hits_test.c c_api/mg_test_utils.cpp) ConfigureCTestMG(MG_CAPI_UNIFORM_NEIGHBOR_SAMPLE c_api/mg_uniform_neighbor_sample_test.c c_api/mg_test_utils.cpp) ConfigureCTestMG(MG_CAPI_TRIANGLE_COUNT c_api/mg_triangle_count_test.c c_api/mg_test_utils.cpp) + ConfigureCTestMG(MG_CAPI_BFS c_api/mg_bfs_test.c c_api/mg_test_utils.cpp) else() message(FATAL_ERROR "OpenMPI NOT found, cannot build MG tests.") endif() diff --git a/cpp/tests/c_api/mg_bfs_test.c b/cpp/tests/c_api/mg_bfs_test.c new file mode 100644 index 00000000000..d1e303fffb1 --- /dev/null +++ b/cpp/tests/c_api/mg_bfs_test.c @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2021-2022, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mg_test_utils.h" /* RUN_TEST */ + +#include +#include + +#include + +typedef int32_t vertex_t; +typedef int32_t edge_t; +typedef float weight_t; + +int generic_bfs_test( + const cugraph_resource_handle_t* p_handle, + vertex_t* h_src, + vertex_t* h_dst, + weight_t* h_wgt, + vertex_t* h_seeds, + vertex_t const* expected_distances, + vertex_t const* expected_predecessors, + size_t num_vertices, + size_t num_edges, + size_t num_seeds, + size_t depth_limit, + bool_t store_transposed) { + int test_ret_value = 0; + + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error; + + cugraph_graph_t* p_graph = NULL; + cugraph_paths_result_t* paths_result = NULL; + cugraph_type_erased_device_array_t* p_sources = NULL; + cugraph_type_erased_device_array_view_t* p_source_view = NULL; + + ret_code = + cugraph_type_erased_device_array_create(p_handle, num_seeds, INT32, &p_sources, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "p_sources create failed."); + + p_source_view = cugraph_type_erased_device_array_view(p_sources); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + p_handle, p_source_view, (byte_t*)h_seeds, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src copy_from_host failed."); + + ret_code = create_mg_test_graph( + p_handle, h_src, h_dst, h_wgt, num_edges, store_transposed, FALSE, &p_graph, &ret_error); + + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "create_mg_test_graph failed."); + + ret_code = cugraph_bfs(p_handle, + p_graph, + p_source_view, + FALSE, + 10000000, + TRUE, + TRUE, + &paths_result, + &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "cugraph_bfs failed."); + + cugraph_type_erased_device_array_view_t* vertices; + cugraph_type_erased_device_array_view_t* distances; + cugraph_type_erased_device_array_view_t* predecessors; + + vertices = cugraph_paths_result_get_vertices(paths_result); + predecessors = cugraph_paths_result_get_predecessors(paths_result); + distances = cugraph_paths_result_get_distances(paths_result); + + vertex_t h_vertices[num_vertices]; + vertex_t h_predecessors[num_vertices]; + vertex_t h_distances[num_vertices]; + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + p_handle, (byte_t*)h_vertices, vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + p_handle, (byte_t*)h_distances, distances, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + p_handle, (byte_t*)h_predecessors, predecessors, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + size_t num_local_vertices = cugraph_type_erased_device_array_view_size(vertices); + + for (int i = 0; (i < num_local_vertices) && (test_ret_value == 0); ++i) { + printf("%d %d\n", expected_distances[h_vertices[i]], h_distances[i]); + TEST_ASSERT(test_ret_value, + expected_distances[h_vertices[i]] == h_distances[i], + "bfs distances don't match"); + + + printf("%d %d\n", expected_predecessors[h_vertices[i]], h_predecessors[i]); + TEST_ASSERT(test_ret_value, + expected_predecessors[h_vertices[i]] == h_predecessors[i], + "bfs predecessors don't match"); + } + + cugraph_paths_result_free(paths_result); + cugraph_mg_graph_free(p_graph); + cugraph_error_free(ret_error); + + return test_ret_value; +} + +int test_bfs(const cugraph_resource_handle_t* p_handle) +{ + size_t num_edges = 8; + size_t num_vertices = 6; + + vertex_t src[] = {0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t dst[] = {1, 3, 4, 0, 1, 3, 5, 5}; + weight_t wgt[] = {0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f}; + vertex_t seeds[] = {0}; + vertex_t expected_distances[] = {0, 1, 2147483647, 2, 2, 3}; + vertex_t expected_predecessors[] = {-1, 0, -1, 1, 1, 3}; + + // Bfs wants store_transposed = FALSE + return generic_bfs_test(p_handle, + src, + dst, + wgt, + seeds, + expected_distances, + expected_predecessors, + num_vertices, + num_edges, + 1, + 10, + FALSE); +} + +int main(int argc, char** argv) +{ + // Set up MPI: + int comm_rank; + int comm_size; + int num_gpus_per_node; + cudaError_t status; + int mpi_status; + cugraph_resource_handle_t* handle = NULL; + cugraph_error_t* ret_error; + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + int prows = 1; + + C_MPI_TRY(MPI_Init(&argc, &argv)); + C_MPI_TRY(MPI_Comm_rank(MPI_COMM_WORLD, &comm_rank)); + C_MPI_TRY(MPI_Comm_size(MPI_COMM_WORLD, &comm_size)); + C_CUDA_TRY(cudaGetDeviceCount(&num_gpus_per_node)); + C_CUDA_TRY(cudaSetDevice(comm_rank % num_gpus_per_node)); + + void* raft_handle = create_raft_handle(prows); + handle = cugraph_create_resource_handle(raft_handle); + int result = 0; + result |= RUN_MG_TEST(test_bfs, handle); + + cugraph_free_resource_handle(handle); + free_raft_handle(raft_handle); + + C_MPI_TRY(MPI_Finalize()); + + return result; +} \ No newline at end of file diff --git a/python/.gitignore b/python/.gitignore index 23a3af14582..49c8f3c3aeb 100644 --- a/python/.gitignore +++ b/python/.gitignore @@ -1,2 +1,3 @@ *.cpp __pycache__ +testing diff --git a/python/cufile.log b/python/cufile.log new file mode 100644 index 00000000000..af134903a5c --- /dev/null +++ b/python/cufile.log @@ -0,0 +1,152 @@ + 17-05-2022 18:03:11:622 [pid=3338 tid=3338] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:03:11:991 [pid=3500 tid=3920] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:03:11:994 [pid=3497 tid=3924] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:03:11:994 [pid=3503 tid=3922] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:03:11:995 [pid=3480 tid=3921] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:03:11:995 [pid=3490 tid=3926] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:03:11:995 [pid=3484 tid=3925] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:03:11:996 [pid=3489 tid=3927] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:03:11:997 [pid=3494 tid=3923] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:41:34:494 [pid=43612 tid=43612] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:41:34:670 [pid=43772 tid=44183] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:41:34:670 [pid=43762 tid=44185] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:41:34:670 [pid=43754 tid=44184] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:41:34:671 [pid=43769 tid=44188] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:41:34:671 [pid=43766 tid=44189] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:41:34:671 [pid=43761 tid=44187] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:41:34:671 [pid=43757 tid=44186] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:41:34:671 [pid=43749 tid=44190] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:45:05:225 [pid=46458 tid=46905] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:45:05:227 [pid=46456 tid=46906] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:45:05:228 [pid=46471 tid=46907] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:45:05:229 [pid=46451 tid=46910] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:45:05:229 [pid=46462 tid=46908] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:45:05:229 [pid=46465 tid=46911] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:45:05:229 [pid=46468 tid=46912] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 18:45:05:229 [pid=46474 tid=46909] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:13:15:122 [pid=69473 tid=69473] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:13:15:274 [pid=69646 tid=70053] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:13:15:274 [pid=69623 tid=70054] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:13:15:275 [pid=69640 tid=70056] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:13:15:275 [pid=69643 tid=70055] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:13:15:275 [pid=69631 tid=70059] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:13:15:276 [pid=69637 tid=70058] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:13:15:276 [pid=69627 tid=70057] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:13:15:277 [pid=69634 tid=70060] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:21:36:350 [pid=74778 tid=74778] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:21:36:555 [pid=74952 tid=75382] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:21:36:559 [pid=74940 tid=75383] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:21:36:559 [pid=74943 tid=75385] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:21:36:560 [pid=74946 tid=75386] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:21:36:560 [pid=74929 tid=75384] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:21:36:561 [pid=74933 tid=75388] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:21:36:561 [pid=74937 tid=75387] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:21:36:562 [pid=74949 tid=75389] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:22:20:322 [pid=75622 tid=75622] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:22:20:475 [pid=75779 tid=76207] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:22:20:475 [pid=75795 tid=76208] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:22:20:476 [pid=75781 tid=76210] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:22:20:476 [pid=75786 tid=76209] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:22:20:477 [pid=75797 tid=76212] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:22:20:477 [pid=75791 tid=76211] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:22:20:478 [pid=75788 tid=76213] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:22:20:479 [pid=75774 tid=76214] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:37:03:382 [pid=6382 tid=6382] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:37:03:527 [pid=6531 tid=6969] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:37:03:527 [pid=6547 tid=6970] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:37:03:528 [pid=6539 tid=6973] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:37:03:528 [pid=6527 tid=6971] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:37:03:529 [pid=6544 tid=6975] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:37:03:530 [pid=6534 tid=6974] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:37:03:530 [pid=6550 tid=6972] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:37:03:530 [pid=6540 tid=6976] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:38:12:678 [pid=7440 tid=7440] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:38:12:854 [pid=7608 tid=8025] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:38:12:856 [pid=7605 tid=8028] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:38:12:857 [pid=7591 tid=8026] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:38:12:857 [pid=7611 tid=8027] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:38:12:857 [pid=7602 tid=8029] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:38:12:857 [pid=7599 tid=8032] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:38:12:858 [pid=7588 tid=8030] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:38:12:858 [pid=7596 tid=8031] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:54:27:169 [pid=15967 tid=15967] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:54:27:362 [pid=16138 tid=16550] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:54:27:363 [pid=16122 tid=16551] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:54:27:363 [pid=16135 tid=16552] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:54:27:363 [pid=16129 tid=16554] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:54:27:363 [pid=16126 tid=16553] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:54:27:363 [pid=16132 tid=16555] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:54:27:364 [pid=16120 tid=16557] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 19:54:27:365 [pid=16115 tid=16556] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:08:51:697 [pid=24245 tid=24245] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:08:51:888 [pid=24420 tid=24844] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:08:51:888 [pid=24408 tid=24845] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:08:51:888 [pid=24404 tid=24846] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:08:51:889 [pid=24402 tid=24849] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:08:51:889 [pid=24414 tid=24847] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:08:51:889 [pid=24397 tid=24848] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:08:51:890 [pid=24411 tid=24850] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:08:51:891 [pid=24417 tid=24851] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:21:26:220 [pid=31544 tid=31544] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:21:26:434 [pid=31693 tid=32115] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:21:26:436 [pid=31704 tid=32116] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:21:26:436 [pid=31695 tid=32117] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:21:26:436 [pid=31698 tid=32118] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:21:26:437 [pid=31685 tid=32119] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:21:26:437 [pid=31681 tid=32120] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:21:26:437 [pid=31688 tid=32121] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:21:26:438 [pid=31701 tid=32122] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:43:26:771 [pid=43701 tid=43701] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:43:26:967 [pid=43858 tid=44270] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:43:26:967 [pid=43850 tid=44269] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:43:26:969 [pid=43838 tid=44272] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:43:26:969 [pid=43852 tid=44271] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:43:26:969 [pid=43847 tid=44275] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:43:26:969 [pid=43855 tid=44273] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:43:26:970 [pid=43861 tid=44274] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:43:26:971 [pid=43842 tid=44276] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:55:08:49 [pid=50394 tid=50394] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:55:08:246 [pid=50543 tid=50970] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:55:08:250 [pid=50546 tid=50971] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:55:08:250 [pid=50552 tid=50973] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:55:08:250 [pid=50549 tid=50972] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:55:08:251 [pid=50529 tid=50975] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:55:08:251 [pid=50537 tid=50976] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:55:08:251 [pid=50532 tid=50977] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 20:55:08:251 [pid=50540 tid=50974] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:11:17:214 [pid=59558 tid=59558] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:11:17:395 [pid=59733 tid=60154] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:11:17:397 [pid=59727 tid=60155] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:11:17:397 [pid=59714 tid=60158] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:11:17:398 [pid=59723 tid=60156] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:11:17:398 [pid=59721 tid=60157] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:11:17:398 [pid=59730 tid=60159] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:11:17:398 [pid=59718 tid=60161] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:11:17:398 [pid=59710 tid=60160] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:14:56:768 [pid=62202 tid=62202] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:14:56:947 [pid=62368 tid=62787] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:14:56:950 [pid=62366 tid=62789] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:14:56:950 [pid=62357 tid=62788] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:14:56:950 [pid=62362 tid=62790] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:14:56:951 [pid=62374 tid=62791] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:14:56:951 [pid=62371 tid=62792] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:14:56:953 [pid=62377 tid=62794] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:14:56:953 [pid=62354 tid=62793] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:17:07:458 [pid=63920 tid=63920] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:17:07:643 [pid=64090 tid=64504] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:17:07:646 [pid=64084 tid=64506] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:17:07:646 [pid=64070 tid=64509] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:17:07:646 [pid=64081 tid=64507] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:17:07:646 [pid=64093 tid=64505] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:17:07:646 [pid=64075 tid=64508] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:17:07:647 [pid=64087 tid=64510] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:17:07:648 [pid=64077 tid=64511] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:19:32:30 [pid=65731 tid=65731] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:19:32:213 [pid=65888 tid=66330] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:19:32:213 [pid=65899 tid=66329] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:19:32:214 [pid=65911 tid=66331] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:19:32:214 [pid=65902 tid=66332] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:19:32:215 [pid=65895 tid=66333] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:19:32:215 [pid=65908 tid=66335] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:19:32:215 [pid=65905 tid=66334] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:19:32:217 [pid=65893 tid=66336] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded diff --git a/python/cugraph/cugraph/dask/traversal/bfs.py b/python/cugraph/cugraph/dask/traversal/bfs.py index b5299085e09..d444571bb49 100644 --- a/python/cugraph/cugraph/dask/traversal/bfs.py +++ b/python/cugraph/cugraph/dask/traversal/bfs.py @@ -27,36 +27,6 @@ import cudf import dask_cudf - -def call_bfs(sID, - data, - src_col_name, - dst_col_name, - num_verts, - num_edges, - vertex_partition_offsets, - aggregate_segment_offsets, - start, - depth_limit, - return_distances): - wid = Comms.get_worker_id(sID) - handle = Comms.get_handle(sID) - local_size = len(aggregate_segment_offsets) // Comms.get_n_workers(sID) - segment_offsets = \ - aggregate_segment_offsets[local_size * wid: local_size * (wid + 1)] - return mg_bfs.mg_bfs(data[0], - src_col_name, - dst_col_name, - num_verts, - num_edges, - vertex_partition_offsets, - wid, - handle, - segment_offsets, - start, - depth_limit, - return_distances) - def _call_plc_mg_bfs( sID, data, @@ -77,35 +47,52 @@ def _call_plc_mg_bfs( if 'value' in data[0].columns \ else cudf.Series((srcs + 1) / (srcs + 1), dtype='float32') - mg = MGGraph( resource_handle = resource_handle, graph_properties = GraphProperties(is_symmetric=False, is_multigraph=False), src_array = srcs, dst_array = dsts, - weights = weights, + weight_array = weights, store_transposed = False, num_edges = num_edges, do_expensive_check = do_expensive_check ) - distances, predecessors, vertices = \ + print({ + 'resource_handle': resource_handle, + 'mg': mg, + 'sources': f'length {len(sources)}, type {type(sources)}', + 'direction_optimizing': direction_optimizing, + 'depth_limit': depth_limit, + 'return_predecessors': return_predecessors, + 'do_expensive_check': do_expensive_check, + 'data': data, + 'sources': sources + }) + + res = \ pylibcugraph_bfs( - resource_handle, + resource_handle, mg, - sources, + cudf.Series(sources, dtype='int32'), direction_optimizing, - depth_limit if depth_limit is not None else -1, + depth_limit if depth_limit is not None else 0, return_predecessors, do_expensive_check ) - return cudf.DataFrame({ - 'distance': cudf.Series(distances), - 'vertex': cudf.Series(vertices), - 'predecessor': cudf.Series(predecessors), - }) + return res +def convert_to_cudf(cp_arrays): + """ + create a cudf DataFrame from cupy arrays + """ + cupy_distances, cupy_predecessors, cupy_vertices = cp_arrays + df = cudf.DataFrame() + df["vertex"] = cupy_vertices + df["distance"] = cupy_distances + df["predecessor"] = cupy_predecessors + return df def bfs(input_graph, start, @@ -166,7 +153,7 @@ def bfs(input_graph, client = default_client() - input_graph.compute_renumber_edge_list(transposed=False) + input_graph.compute_renumber_edge_list(transposed=False,) ddf = input_graph.edgelist.edgelist_df vertex_partition_offsets = get_vertex_partition_offsets(input_graph) num_verts = vertex_partition_offsets.iloc[-1] @@ -178,69 +165,45 @@ def df_merge(df, tmp_df, tmp_col_names): x = df[0].merge(tmp_df, on=tmp_col_names, how='inner') return x['global_id'] - if input_graph.renumbered: - src_col_name = input_graph.renumber_map.renumbered_src_col_name - dst_col_name = input_graph.renumber_map.renumbered_dst_col_name - renumber_ddf = input_graph.renumber_map.implementation.ddf - col_names = input_graph.renumber_map.implementation.col_names - if isinstance(start, - dask_cudf.DataFrame) or isinstance(start, - cudf.DataFrame): - tmp_df = start - tmp_col_names = start.columns - else: - tmp_df = cudf.DataFrame() - tmp_df["0"] = cudf.Series(start) - tmp_col_names = ["0"] - - original_start_len = len(tmp_df) - - tmp_ddf = tmp_df[tmp_col_names].rename( - columns=dict(zip(tmp_col_names, col_names))) - for name in col_names: - tmp_ddf[name] = tmp_ddf[name].astype(renumber_ddf[name].dtype) - renumber_data = get_distributed_data(renumber_ddf) - start = [client.submit(df_merge, - wf[1], - tmp_ddf, - col_names, - workers=[wf[0]]) - for idx, wf in enumerate(renumber_data.worker_to_parts.items() - ) - ] - - def count_src(df): - return len(df) - - count_src_results = client.map(count_src, start) - cg = client.gather(count_src_results) - if sum(cg) < original_start_len: - raise ValueError('At least one start vertex provided was invalid') - + src_col_name = input_graph.renumber_map.renumbered_src_col_name + dst_col_name = input_graph.renumber_map.renumbered_dst_col_name + renumber_ddf = input_graph.renumber_map.implementation.ddf + col_names = input_graph.renumber_map.implementation.col_names + if isinstance(start, + dask_cudf.DataFrame) or isinstance(start, + cudf.DataFrame): + tmp_df = start + tmp_col_names = start.columns else: - # If the input graph was created with renumbering disabled (Graph(..., - # renumber=False), the above compute_renumber_edge_list() call will not - # perform a renumber step and the renumber_map will not have src/dst - # col names. In that case, the src/dst values specified when reading - # the edgelist dataframe are to be used, but only if they were single - # string values (ie. not a list representing multi-columns). - if isinstance(input_graph.source_columns, Iterable): - raise RuntimeError("input_graph was not renumbered but has a " - "non-string source column name (got: " - f"{input_graph.source_columns}). Re-create " - "input_graph with either renumbering enabled " - "or a source column specified as a string.") - if isinstance(input_graph.destination_columns, Iterable): - raise RuntimeError("input_graph was not renumbered but has a " - "non-string destination column name (got: " - f"{input_graph.destination_columns}). " - "Re-create input_graph with either renumbering " - "enabled or a destination column specified as " - "a string.") - src_col_name = input_graph.source_columns - dst_col_name = input_graph.destination_columns - - result = [client.submit( + tmp_df = cudf.DataFrame() + tmp_df["0"] = cudf.Series(start) + tmp_col_names = ["0"] + + original_start_len = len(tmp_df) + + tmp_ddf = tmp_df[tmp_col_names].rename( + columns=dict(zip(tmp_col_names, col_names))) + for name in col_names: + tmp_ddf[name] = tmp_ddf[name].astype(renumber_ddf[name].dtype) + renumber_data = get_distributed_data(renumber_ddf) + start = [client.submit(df_merge, + wf[1], + tmp_ddf, + col_names, + workers=[wf[0]]) + for idx, wf in enumerate(renumber_data.worker_to_parts.items() + ) + ] + + def count_src(df): + return len(df) + + count_src_results = client.map(count_src, start) + cg = client.gather(count_src_results) + if sum(cg) < original_start_len: + raise ValueError('At least one start vertex provided was invalid') + + cupy_result = [client.submit( _call_plc_mg_bfs, Comms.get_session_id(), wf[1], @@ -250,12 +213,22 @@ def count_src(df): dst_col_name, num_edges, False, - False, + True, return_distances, workers=[wf[0]]) for idx, wf in enumerate(data.worker_to_parts.items())] - wait(result) - ddf = dask_cudf.from_delayed(result) + wait(cupy_result) + print('wait 1') + print('wait 2') + cudf_result = [client.submit(convert_to_cudf, + cp_arrays) + for cp_arrays in cupy_result] + print('wait 3') + wait(cudf_result) + print('wait 4') + + ddf = dask_cudf.from_delayed(cudf_result) + print('wait 5') if input_graph.renumbered: ddf = input_graph.unrenumber(ddf, 'vertex') diff --git a/python/cugraph/cugraph/traversal/sssp_wrapper.pyx b/python/cugraph/cugraph/traversal/sssp_wrapper.pyx index 46966cd3e99..6972cd40b68 100644 --- a/python/cugraph/cugraph/traversal/sssp_wrapper.pyx +++ b/python/cugraph/cugraph/traversal/sssp_wrapper.pyx @@ -17,7 +17,6 @@ # cython: language_level = 3 cimport cugraph.traversal.sssp as c_sssp -cimport cugraph.traversal.bfs as c_bfs from cugraph.structure.graph_utilities cimport * from cugraph.structure import graph_primtypes_wrapper from libcpp cimport bool diff --git a/python/pylibcugraph/pylibcugraph/bfs.pyx b/python/pylibcugraph/pylibcugraph/bfs.pyx index 7eabff95749..eb30b898736 100644 --- a/python/pylibcugraph/pylibcugraph/bfs.pyx +++ b/python/pylibcugraph/pylibcugraph/bfs.pyx @@ -39,7 +39,7 @@ from pylibcugraph.graphs cimport ( ) def bfs(ResourceHandle handle, _GPUGraph graph, - sources, bool_t direction_optimizing, int64_t depth_limit, + sources, bool_t direction_optimizing, size_t depth_limit, bool_t compute_predecessors, bool_t do_expensive_check): try: import cupy @@ -48,7 +48,7 @@ def bfs(ResourceHandle handle, _GPUGraph graph, "be imported") assert_CAI_type(sources, "sources") - if depth_limit < 0: + if depth_limit == 0: depth_limit = INT_MAX cdef cugraph_resource_handle_t* c_resource_handle_ptr = \ @@ -60,6 +60,7 @@ def bfs(ResourceHandle handle, _GPUGraph graph, cdef uintptr_t cai_sources_ptr = \ sources.__cuda_array_interface__["data"][0] + cdef cugraph_type_erased_device_array_view_t* sources_view_ptr = \ cugraph_type_erased_device_array_view_create( cai_sources_ptr, @@ -81,9 +82,6 @@ def bfs(ResourceHandle handle, _GPUGraph graph, ) assert_success(error_code, error_ptr, "cugraph_bfs") - # deallocate the no-longer needed sources array - cugraph_type_erased_device_array_view_free(sources_view_ptr) - # Extract individual device array pointers from result cdef cugraph_type_erased_device_array_view_t* distances_ptr = \ cugraph_paths_result_get_distances(result_ptr) diff --git a/testing/cufile.log b/testing/cufile.log new file mode 100644 index 00000000000..8bea5444044 --- /dev/null +++ b/testing/cufile.log @@ -0,0 +1,480 @@ + 17-05-2022 21:40:47:52 [pid=80962 tid=81356] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:40:47:53 [pid=80976 tid=81358] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:40:47:53 [pid=80967 tid=81357] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:40:47:54 [pid=80971 tid=81359] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:40:47:55 [pid=80968 tid=81360] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:40:47:55 [pid=80958 tid=81362] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:40:47:55 [pid=80979 tid=81361] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:40:47:55 [pid=80969 tid=81363] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:41:37:200 [pid=81717 tid=432] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:41:37:204 [pid=81693 tid=433] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:41:37:205 [pid=81710 tid=434] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:41:37:206 [pid=81719 tid=435] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:41:37:206 [pid=81708 tid=439] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:41:37:206 [pid=81699 tid=438] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:41:37:207 [pid=81703 tid=436] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:41:37:207 [pid=81686 tid=437] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:42:39:122 [pid=884 tid=1228] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:42:39:123 [pid=872 tid=1230] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:42:39:123 [pid=877 tid=1231] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:42:39:123 [pid=893 tid=1233] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:42:39:123 [pid=880 tid=1229] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:42:39:124 [pid=890 tid=1234] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:42:39:124 [pid=888 tid=1232] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:42:39:125 [pid=870 tid=1235] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:57:55:907 [pid=9582 tid=9944] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:57:55:908 [pid=9588 tid=9945] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:57:55:909 [pid=9576 tid=9946] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:57:55:909 [pid=9585 tid=9948] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:57:55:909 [pid=9579 tid=9947] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:57:55:909 [pid=9569 tid=9949] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:57:55:910 [pid=9565 tid=9950] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 21:57:55:910 [pid=9574 tid=9951] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:01:31:982 [pid=11929 tid=12279] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:01:31:982 [pid=11920 tid=12278] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:01:31:984 [pid=11915 tid=12281] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:01:31:984 [pid=11924 tid=12280] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:01:31:984 [pid=11918 tid=12282] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:01:31:985 [pid=11911 tid=12284] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:01:31:985 [pid=11926 tid=12285] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:01:31:985 [pid=11906 tid=12283] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:03:36:564 [pid=13364 tid=13750] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:03:36:564 [pid=13358 tid=13751] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:03:36:566 [pid=13346 tid=13752] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:03:36:566 [pid=13354 tid=13754] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:03:36:567 [pid=13341 tid=13753] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:03:36:568 [pid=13361 tid=13755] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:03:36:568 [pid=13353 tid=13756] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:03:36:568 [pid=13349 tid=13757] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:22:38:109 [pid=765 tid=1124] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:22:38:109 [pid=759 tid=1126] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:22:38:109 [pid=776 tid=1127] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:22:38:109 [pid=770 tid=1125] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:22:38:109 [pid=754 tid=1128] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:22:38:110 [pid=767 tid=1129] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:22:38:111 [pid=761 tid=1131] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:22:38:111 [pid=773 tid=1130] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:37:22:984 [pid=9037 tid=9437] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:37:22:987 [pid=9044 tid=9438] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:37:22:988 [pid=9051 tid=9443] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:37:22:988 [pid=9056 tid=9439] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:37:22:988 [pid=9042 tid=9441] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:37:22:988 [pid=9059 tid=9440] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:37:22:989 [pid=9053 tid=9444] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:37:22:989 [pid=9048 tid=9442] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:44:43:819 [pid=14440 tid=14774] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:44:43:824 [pid=14438 tid=14775] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:44:43:824 [pid=14434 tid=14776] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:44:43:824 [pid=14421 tid=14777] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:44:43:825 [pid=14429 tid=14781] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:44:43:825 [pid=14424 tid=14779] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:44:43:826 [pid=14443 tid=14778] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:44:43:826 [pid=14431 tid=14780] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:50:46:340 [pid=18062 tid=18415] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:50:46:341 [pid=18076 tid=18416] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:50:46:342 [pid=18066 tid=18418] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:50:46:342 [pid=18070 tid=18417] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:50:46:342 [pid=18081 tid=18419] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:50:46:343 [pid=18071 tid=18420] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:50:46:344 [pid=18078 tid=18421] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:50:46:344 [pid=18084 tid=18422] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:54:24:705 [pid=20369 tid=20749] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:54:24:710 [pid=20366 tid=20750] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:54:24:711 [pid=20351 tid=20753] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:54:24:711 [pid=20346 tid=20752] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:54:24:711 [pid=20355 tid=20751] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:54:24:712 [pid=20358 tid=20754] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:54:24:712 [pid=20360 tid=20755] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:54:24:712 [pid=20363 tid=20756] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:57:14:210 [pid=22242 tid=22584] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:57:14:211 [pid=22238 tid=22586] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:57:14:211 [pid=22225 tid=22585] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:57:14:212 [pid=22233 tid=22589] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:57:14:213 [pid=22228 tid=22587] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:57:14:213 [pid=22236 tid=22588] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:57:14:214 [pid=22248 tid=22591] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 22:57:14:215 [pid=22245 tid=22590] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:00:41:835 [pid=24294 tid=24659] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:00:41:838 [pid=24304 tid=24660] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:00:41:839 [pid=24316 tid=24661] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:00:41:839 [pid=24297 tid=24662] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:00:41:840 [pid=24302 tid=24663] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:00:41:840 [pid=24313 tid=24664] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:00:41:841 [pid=24308 tid=24666] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:00:41:841 [pid=24309 tid=24665] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:02:48:381 [pid=741 tid=1079] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:02:48:382 [pid=736 tid=1080] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:02:48:382 [pid=752 tid=1081] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:02:48:382 [pid=733 tid=1084] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:02:48:382 [pid=749 tid=1083] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:02:48:382 [pid=746 tid=1086] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:02:48:382 [pid=743 tid=1082] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:02:48:383 [pid=755 tid=1085] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:04:46:755 [pid=2219 tid=2562] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:04:46:755 [pid=2213 tid=2561] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:04:46:759 [pid=2209 tid=2565] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:04:46:760 [pid=2222 tid=2563] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:04:46:760 [pid=2202 tid=2564] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:04:46:760 [pid=2206 tid=2566] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:04:46:760 [pid=2224 tid=2567] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:04:46:762 [pid=2216 tid=2568] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:12:29:196 [pid=6696 tid=7058] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:12:29:197 [pid=6709 tid=7065] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:12:29:197 [pid=6687 tid=7060] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:12:29:197 [pid=6697 tid=7059] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:12:29:197 [pid=6704 tid=7061] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:12:29:198 [pid=6682 tid=7062] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:12:29:198 [pid=6706 tid=7064] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:12:29:198 [pid=6701 tid=7063] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:22:31:944 [pid=12411 tid=12763] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:22:31:945 [pid=12409 tid=12764] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:22:31:946 [pid=12393 tid=12766] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:22:31:946 [pid=12389 tid=12765] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:22:31:947 [pid=12405 tid=12767] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:22:31:947 [pid=12403 tid=12768] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:22:31:948 [pid=12398 tid=12770] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:22:31:948 [pid=12396 tid=12769] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:25:42:8 [pid=14490 tid=14865] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:25:42:8 [pid=14502 tid=14864] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:25:42:8 [pid=14492 tid=14863] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:25:42:9 [pid=14498 tid=14867] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:25:42:9 [pid=14504 tid=14866] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:25:42:9 [pid=14495 tid=14868] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:25:42:9 [pid=14482 tid=14870] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:25:42:9 [pid=14486 tid=14869] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:42:34:810 [pid=23802 tid=24177] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:42:34:810 [pid=23791 tid=24180] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:42:34:810 [pid=23808 tid=24181] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:42:34:810 [pid=23798 tid=24183] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:42:34:810 [pid=23811 tid=24182] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:42:34:810 [pid=23805 tid=24178] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:42:34:810 [pid=23794 tid=24176] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:42:34:811 [pid=23813 tid=24179] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:46:58:752 [pid=26563 tid=26929] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:46:58:754 [pid=26574 tid=26931] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:46:58:754 [pid=26572 tid=26933] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:46:58:755 [pid=26581 tid=26932] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:46:58:755 [pid=26577 tid=26930] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:46:58:755 [pid=26583 tid=26935] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:46:58:755 [pid=26566 tid=26934] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:46:58:757 [pid=26584 tid=26936] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:50:12:847 [pid=28661 tid=29053] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:50:12:850 [pid=28670 tid=29054] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:50:12:851 [pid=28668 tid=29055] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:50:12:851 [pid=28664 tid=29056] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:50:12:851 [pid=28655 tid=29057] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:50:12:852 [pid=28658 tid=29058] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:50:12:853 [pid=28651 tid=29059] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:50:12:853 [pid=28648 tid=29060] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:55:42:202 [pid=32311 tid=32708] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:55:42:202 [pid=32297 tid=32706] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:55:42:202 [pid=32316 tid=32707] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:55:42:203 [pid=32302 tid=32709] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:55:42:204 [pid=32294 tid=32712] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:55:42:204 [pid=32313 tid=32710] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:55:42:204 [pid=32305 tid=32711] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 17-05-2022 23:55:42:204 [pid=32308 tid=32713] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:02:42:130 [pid=36342 tid=36777] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:02:42:132 [pid=36337 tid=36778] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:02:42:132 [pid=36352 tid=36779] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:02:42:133 [pid=36346 tid=36780] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:02:42:133 [pid=36359 tid=36781] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:02:42:133 [pid=36354 tid=36782] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:02:42:134 [pid=36363 tid=36783] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:02:42:134 [pid=36348 tid=36784] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:11:54:868 [pid=42479 tid=42838] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:11:54:868 [pid=42469 tid=42839] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:11:54:869 [pid=42490 tid=42841] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:11:54:869 [pid=42475 tid=42840] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:11:54:871 [pid=42480 tid=42844] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:11:54:871 [pid=42485 tid=42842] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:11:54:871 [pid=42481 tid=42843] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:11:54:871 [pid=42489 tid=42845] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:14:18:596 [pid=44179 tid=44528] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:14:18:601 [pid=44175 tid=44530] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:14:18:601 [pid=44190 tid=44531] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:14:18:602 [pid=44171 tid=44533] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:14:18:602 [pid=44187 tid=44532] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:14:18:602 [pid=44185 tid=44534] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:14:18:602 [pid=44181 tid=44529] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:14:18:602 [pid=44193 tid=44535] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:17:58:669 [pid=46538 tid=46935] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:17:58:671 [pid=46547 tid=46936] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:17:58:672 [pid=46560 tid=46938] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:17:58:672 [pid=46533 tid=46940] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:17:58:672 [pid=46554 tid=46937] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:17:58:672 [pid=46541 tid=46939] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:17:58:672 [pid=46544 tid=46941] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:17:58:673 [pid=46550 tid=46942] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:23:04:395 [pid=49642 tid=50057] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:23:04:397 [pid=49622 tid=50059] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:23:04:397 [pid=49623 tid=50058] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:23:04:397 [pid=49636 tid=50061] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:23:04:397 [pid=49631 tid=50062] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:23:04:397 [pid=49639 tid=50060] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:23:04:398 [pid=49616 tid=50063] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:23:04:399 [pid=49613 tid=50064] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:25:02:949 [pid=51073 tid=51497] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:25:02:951 [pid=51085 tid=51498] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:25:02:951 [pid=51067 tid=51499] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:25:02:951 [pid=51063 tid=51501] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:25:02:951 [pid=51082 tid=51500] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:25:02:951 [pid=51072 tid=51503] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:25:02:952 [pid=51077 tid=51502] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:25:02:953 [pid=51079 tid=51504] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:28:15:196 [pid=53385 tid=53810] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:28:15:196 [pid=53392 tid=53811] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:28:15:198 [pid=53398 tid=53812] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:28:15:198 [pid=53396 tid=53816] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:28:15:198 [pid=53384 tid=53813] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:28:15:198 [pid=53377 tid=53814] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:28:15:198 [pid=53379 tid=53815] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 00:28:15:199 [pid=53372 tid=53817] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:44:28:218 [pid=77209 tid=77611] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:44:28:219 [pid=77214 tid=77612] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:44:28:220 [pid=77221 tid=77613] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:44:28:221 [pid=77217 tid=77614] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:44:28:221 [pid=77205 tid=77615] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:44:28:221 [pid=77212 tid=77616] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:44:28:222 [pid=77223 tid=77617] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:44:28:222 [pid=77201 tid=77618] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:48:05:448 [pid=79550 tid=79941] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:48:05:448 [pid=79545 tid=79940] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:48:05:450 [pid=79535 tid=79942] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:48:05:450 [pid=79553 tid=79943] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:48:05:451 [pid=79531 tid=79944] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:48:05:451 [pid=79547 tid=79945] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:48:05:452 [pid=79539 tid=79946] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:48:05:454 [pid=79542 tid=79947] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:52:05:661 [pid=334 tid=772] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:52:05:664 [pid=355 tid=773] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:52:05:665 [pid=352 tid=775] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:52:05:665 [pid=338 tid=774] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:52:05:666 [pid=358 tid=776] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:52:05:666 [pid=350 tid=777] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:52:05:667 [pid=340 tid=778] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:52:05:667 [pid=347 tid=779] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:56:35:733 [pid=2851 tid=3285] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:56:35:733 [pid=2857 tid=3286] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:56:35:733 [pid=2867 tid=3287] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:56:35:734 [pid=2871 tid=3288] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:56:35:735 [pid=2865 tid=3292] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:56:35:735 [pid=2855 tid=3291] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:56:35:737 [pid=2862 tid=3289] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 02:56:35:737 [pid=2873 tid=3290] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:02:42:426 [pid=6301 tid=6731] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:02:42:427 [pid=6290 tid=6732] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:02:42:428 [pid=6304 tid=6733] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:02:42:428 [pid=6286 tid=6734] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:02:42:429 [pid=6298 tid=6736] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:02:42:429 [pid=6282 tid=6735] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:02:42:429 [pid=6294 tid=6737] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:02:42:430 [pid=6293 tid=6738] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:05:20:738 [pid=8267 tid=8606] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:05:20:741 [pid=8261 tid=8609] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:05:20:741 [pid=8270 tid=8607] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:05:20:741 [pid=8277 tid=8611] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:05:20:742 [pid=8284 tid=8608] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:05:20:742 [pid=8257 tid=8610] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:05:20:742 [pid=8265 tid=8612] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:05:20:743 [pid=8280 tid=8613] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:10:43:600 [pid=18195 tid=18568] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:10:43:600 [pid=18205 tid=18572] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:10:43:601 [pid=18202 tid=18569] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:10:43:601 [pid=18185 tid=18571] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:10:43:601 [pid=18201 tid=18570] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:10:43:601 [pid=18183 tid=18573] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:10:43:603 [pid=18196 tid=18574] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:10:43:603 [pid=18191 tid=18575] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:14:15:563 [pid=20251 tid=20510] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:14:15:716 [pid=20244 tid=20514] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:14:15:738 [pid=20237 tid=20515] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:14:15:772 [pid=20256 tid=20516] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:14:15:790 [pid=20234 tid=20517] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:14:15:812 [pid=20248 tid=20518] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:14:15:824 [pid=20259 tid=20519] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:14:15:834 [pid=20231 tid=20520] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:19:32:698 [pid=23570 tid=23968] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:19:32:700 [pid=23578 tid=23970] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:19:32:701 [pid=23559 tid=23972] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:19:32:701 [pid=23575 tid=23973] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:19:32:701 [pid=23565 tid=23971] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:19:32:702 [pid=23553 tid=23969] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:19:32:703 [pid=23557 tid=23974] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:19:32:704 [pid=23564 tid=23975] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:22:14:798 [pid=25386 tid=25816] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:22:14:801 [pid=25393 tid=25818] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:22:14:801 [pid=25399 tid=25819] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:22:14:801 [pid=25388 tid=25817] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:22:14:801 [pid=25396 tid=25821] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:22:14:802 [pid=25407 tid=25820] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:22:14:802 [pid=25404 tid=25822] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:22:14:803 [pid=25402 tid=25823] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:25:02:610 [pid=27235 tid=27638] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:25:02:612 [pid=27210 tid=27639] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:25:02:613 [pid=27223 tid=27640] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:25:02:614 [pid=27227 tid=27641] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:25:02:614 [pid=27231 tid=27642] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:25:02:614 [pid=27220 tid=27645] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:25:02:614 [pid=27213 tid=27643] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:25:02:614 [pid=27217 tid=27644] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:28:05:236 [pid=29145 tid=29593] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:28:05:240 [pid=29148 tid=29594] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:28:05:241 [pid=29153 tid=29595] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:28:05:241 [pid=29156 tid=29596] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:28:05:241 [pid=29150 tid=29597] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:28:05:241 [pid=29134 tid=29598] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:28:05:242 [pid=29138 tid=29600] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:28:05:242 [pid=29142 tid=29599] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:32:43:258 [pid=32655 tid=32989] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:32:43:258 [pid=32652 tid=32990] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:32:43:259 [pid=32649 tid=32991] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:32:43:260 [pid=32642 tid=32992] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:32:43:260 [pid=32657 tid=32995] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:32:43:260 [pid=32638 tid=32993] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:32:43:260 [pid=32645 tid=32994] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:32:43:261 [pid=32661 tid=32996] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:34:41:896 [pid=34038 tid=34430] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:34:41:896 [pid=34043 tid=34428] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:34:41:897 [pid=34046 tid=34429] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:34:41:898 [pid=34029 tid=34431] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:34:41:898 [pid=34040 tid=34432] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:34:41:898 [pid=34032 tid=34433] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:34:41:898 [pid=34025 tid=34434] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:34:41:899 [pid=34021 tid=34435] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:36:41:608 [pid=35324 tid=35724] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:36:41:613 [pid=35330 tid=35728] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:36:41:613 [pid=35348 tid=35725] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:36:41:613 [pid=35342 tid=35726] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:36:41:614 [pid=35346 tid=35727] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:36:41:614 [pid=35338 tid=35729] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:36:41:614 [pid=35337 tid=35730] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:36:41:615 [pid=35334 tid=35731] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:38:47:542 [pid=36799 tid=37215] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:38:47:543 [pid=36814 tid=37216] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:38:47:543 [pid=36811 tid=37214] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:38:47:543 [pid=36817 tid=37218] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:38:47:543 [pid=36802 tid=37217] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:38:47:543 [pid=36805 tid=37219] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:38:47:544 [pid=36808 tid=37220] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:38:47:545 [pid=36795 tid=37221] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:44:33:312 [pid=40804 tid=41135] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:44:33:312 [pid=40781 tid=41137] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:44:33:312 [pid=40808 tid=41136] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:44:33:313 [pid=40801 tid=41138] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:44:33:314 [pid=40789 tid=41139] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:44:33:314 [pid=40798 tid=41140] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:44:33:314 [pid=40792 tid=41141] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 03:44:33:316 [pid=40795 tid=41142] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:55:13:531 [pid=71995 tid=72369] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:55:13:531 [pid=71999 tid=72373] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:55:13:531 [pid=72011 tid=72368] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:55:13:531 [pid=72005 tid=72372] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:55:13:531 [pid=72007 tid=72370] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:55:13:531 [pid=71990 tid=72375] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:55:13:531 [pid=72001 tid=72374] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:55:13:531 [pid=72013 tid=72371] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:56:58:187 [pid=73488 tid=73827] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:56:58:187 [pid=73471 tid=73828] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:56:58:188 [pid=73477 tid=73830] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:56:58:188 [pid=73498 tid=73829] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:56:58:189 [pid=73481 tid=73831] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:56:58:189 [pid=73472 tid=73832] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:56:58:189 [pid=73492 tid=73833] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:56:58:189 [pid=73483 tid=73834] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:59:44:910 [pid=75350 tid=75690] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:59:44:913 [pid=75354 tid=75691] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:59:44:913 [pid=75347 tid=75692] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:59:44:914 [pid=75342 tid=75696] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:59:44:915 [pid=75345 tid=75693] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:59:44:916 [pid=75334 tid=75695] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:59:44:916 [pid=75337 tid=75694] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 12:59:44:917 [pid=75356 tid=75697] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:02:42:583 [pid=77681 tid=78034] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:02:42:583 [pid=77692 tid=78036] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:02:42:583 [pid=77691 tid=78035] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:02:42:584 [pid=77699 tid=78038] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:02:42:584 [pid=77684 tid=78039] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:02:42:584 [pid=77676 tid=78037] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:02:42:584 [pid=77688 tid=78040] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:02:42:585 [pid=77696 tid=78041] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:05:04:504 [pid=79223 tid=79576] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:05:04:507 [pid=79218 tid=79577] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:05:04:509 [pid=79232 tid=79578] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:05:04:509 [pid=79210 tid=79579] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:05:04:509 [pid=79214 tid=79582] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:05:04:509 [pid=79226 tid=79581] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:05:04:510 [pid=79219 tid=79580] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:05:04:510 [pid=79229 tid=79583] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:07:52:128 [pid=81383 tid=81774] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:07:52:128 [pid=81406 tid=81778] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:07:52:129 [pid=81400 tid=81775] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:07:52:129 [pid=81403 tid=81776] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:07:52:129 [pid=81398 tid=81777] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:07:52:129 [pid=81395 tid=81779] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:07:52:131 [pid=81389 tid=81780] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:07:52:131 [pid=81391 tid=81781] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:27:20:252 [pid=10465 tid=10867] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:27:20:253 [pid=10466 tid=10870] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:27:20:253 [pid=10454 tid=10869] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:27:20:253 [pid=10459 tid=10868] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:27:20:253 [pid=10471 tid=10871] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:27:20:255 [pid=10477 tid=10872] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:27:20:255 [pid=10462 tid=10874] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:27:20:255 [pid=10480 tid=10873] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:35:11:429 [pid=15345 tid=15704] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:35:11:429 [pid=15358 tid=15705] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:35:11:429 [pid=15361 tid=15709] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:35:11:429 [pid=15352 tid=15706] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:35:11:429 [pid=15348 tid=15710] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:35:11:429 [pid=15341 tid=15707] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:35:11:430 [pid=15354 tid=15708] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 13:35:11:430 [pid=15363 tid=15711] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 14:08:16:795 [pid=28767 tid=29148] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 14:08:16:795 [pid=28758 tid=29150] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 14:08:16:795 [pid=28778 tid=29152] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 14:08:16:795 [pid=28773 tid=29154] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 14:08:16:795 [pid=28774 tid=29153] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 14:08:16:795 [pid=28762 tid=29151] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 14:08:16:795 [pid=28780 tid=29149] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 14:08:16:795 [pid=28768 tid=29155] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 14:14:40:101 [pid=32624 tid=33038] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 14:14:40:101 [pid=32608 tid=33036] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 14:14:40:102 [pid=32622 tid=33037] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 14:14:40:102 [pid=32602 tid=33039] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 14:14:40:103 [pid=32613 tid=33040] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 14:14:40:103 [pid=32616 tid=33043] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 14:14:40:103 [pid=32619 tid=33041] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 14:14:40:103 [pid=32606 tid=33042] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 15:58:20:951 [pid=21836 tid=22705] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 15:58:20:952 [pid=21847 tid=22704] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 15:58:20:952 [pid=21841 tid=22703] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 15:58:20:952 [pid=21838 tid=22707] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 15:58:20:952 [pid=21844 tid=22706] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 15:58:20:952 [pid=21825 tid=22708] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 15:58:20:952 [pid=21829 tid=22709] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 15:58:20:953 [pid=21833 tid=22710] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:07:52:420 [pid=13057 tid=13482] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:07:52:422 [pid=13045 tid=13484] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:07:52:422 [pid=13041 tid=13483] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:07:52:423 [pid=13049 tid=13487] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:07:52:423 [pid=13060 tid=13486] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:07:52:424 [pid=13064 tid=13488] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:07:52:424 [pid=13054 tid=13489] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:07:52:425 [pid=13051 tid=13485] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:09:38:528 [pid=14499 tid=14923] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:09:38:529 [pid=14496 tid=14922] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:09:38:529 [pid=14489 tid=14924] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:09:38:530 [pid=14492 tid=14925] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:09:38:530 [pid=14491 tid=14927] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:09:38:530 [pid=14477 tid=14926] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:09:38:531 [pid=14502 tid=14928] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:09:38:531 [pid=14485 tid=14929] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:10:33:230 [pid=15322 tid=15762] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:10:33:231 [pid=15334 tid=15763] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:10:33:232 [pid=15317 tid=15765] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:10:33:233 [pid=15326 tid=15766] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:10:33:233 [pid=15331 tid=15767] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:10:33:233 [pid=15328 tid=15764] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:10:33:235 [pid=15312 tid=15768] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:10:33:235 [pid=15319 tid=15769] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded diff --git a/testing/dask-worker-space/global.lock b/testing/dask-worker-space/global.lock new file mode 100644 index 00000000000..e69de29bb2d diff --git a/testing/dask-worker-space/purge.lock b/testing/dask-worker-space/purge.lock new file mode 100644 index 00000000000..e69de29bb2d diff --git a/testing/dask-worker-space/worker-i66t8r65.dirlock b/testing/dask-worker-space/worker-i66t8r65.dirlock new file mode 100644 index 00000000000..e69de29bb2d diff --git a/testing/dask-worker-space/worker-ifem89p4.dirlock b/testing/dask-worker-space/worker-ifem89p4.dirlock new file mode 100644 index 00000000000..e69de29bb2d diff --git a/testing/dask-worker-space/worker-jvu0j__k.dirlock b/testing/dask-worker-space/worker-jvu0j__k.dirlock new file mode 100644 index 00000000000..e69de29bb2d diff --git a/testing/dask-worker-space/worker-l968y9vr.dirlock b/testing/dask-worker-space/worker-l968y9vr.dirlock new file mode 100644 index 00000000000..e69de29bb2d diff --git a/testing/dask-worker-space/worker-lr81p2a2.dirlock b/testing/dask-worker-space/worker-lr81p2a2.dirlock new file mode 100644 index 00000000000..e69de29bb2d diff --git a/testing/dask-worker-space/worker-wjcu9c0j.dirlock b/testing/dask-worker-space/worker-wjcu9c0j.dirlock new file mode 100644 index 00000000000..e69de29bb2d diff --git a/testing/dask-worker-space/worker-xtysqw6a.dirlock b/testing/dask-worker-space/worker-xtysqw6a.dirlock new file mode 100644 index 00000000000..e69de29bb2d diff --git a/testing/dask-worker-space/worker-ykczxdz3.dirlock b/testing/dask-worker-space/worker-ykczxdz3.dirlock new file mode 100644 index 00000000000..e69de29bb2d diff --git a/testing/testing1.ipynb b/testing/testing1.ipynb new file mode 100644 index 00000000000..07b980c5c24 --- /dev/null +++ b/testing/testing1.ipynb @@ -0,0 +1,125 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2022-05-18 20:09:21,992 - distributed.diskutils - INFO - Found stale lock file and directory '/root/cugraph/testing/dask-worker-space/worker-14xf1dwk', purging\n", + "2022-05-18 20:09:21,992 - distributed.diskutils - INFO - Found stale lock file and directory '/root/cugraph/testing/dask-worker-space/worker-14_bx41d', purging\n", + "2022-05-18 20:09:21,993 - distributed.diskutils - INFO - Found stale lock file and directory '/root/cugraph/testing/dask-worker-space/worker-phli1i9q', purging\n", + "2022-05-18 20:09:21,993 - distributed.diskutils - INFO - Found stale lock file and directory '/root/cugraph/testing/dask-worker-space/worker-ekqa9enz', purging\n", + "2022-05-18 20:09:21,993 - distributed.diskutils - INFO - Found stale lock file and directory '/root/cugraph/testing/dask-worker-space/worker-y77x0ytl', purging\n", + "2022-05-18 20:09:21,993 - distributed.diskutils - INFO - Found stale lock file and directory '/root/cugraph/testing/dask-worker-space/worker-807hal14', purging\n", + "2022-05-18 20:09:21,994 - distributed.diskutils - INFO - Found stale lock file and directory '/root/cugraph/testing/dask-worker-space/worker-sv410h22', purging\n", + "2022-05-18 20:09:21,994 - distributed.diskutils - INFO - Found stale lock file and directory '/root/cugraph/testing/dask-worker-space/worker-myjjhguf', purging\n", + "2022-05-18 20:09:21,994 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", + "2022-05-18 20:09:22,025 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", + "2022-05-18 20:09:22,039 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", + "2022-05-18 20:09:22,044 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", + "2022-05-18 20:09:22,050 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", + "2022-05-18 20:09:22,051 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", + "2022-05-18 20:09:22,054 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", + "2022-05-18 20:09:22,064 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", + "/opt/conda/envs/rapids/lib/python3.8/site-packages/cudf/core/dataframe.py:1055: FutureWarning: The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning.\n", + " return pd.Series(self._dtypes)\n" + ] + }, + { + "ename": "TypeError", + "evalue": "bfs() got an unexpected keyword argument 'do_expensive_check'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/tmp/ipykernel_14334/244951834.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;31m# now run PageRank\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0mpr_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdask_cugraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbfs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mG\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth_limit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdo_expensive_check\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0;31m# All done, clean up\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mTypeError\u001b[0m: bfs() got an unexpected keyword argument 'do_expensive_check'" + ] + } + ], + "source": [ + "from dask.distributed import Client, wait\n", + "from dask_cuda import LocalCUDACluster\n", + "from cugraph.dask.comms import comms\n", + "import cugraph.dask as dask_cugraph\n", + "import cugraph, dask_cudf\n", + "\n", + "input_data_path = '../datasets/karate.csv'\n", + "\n", + "cluster = LocalCUDACluster()\n", + "client = Client(cluster)\n", + "comms.initialize(p2p=True)\n", + "\n", + "# Helper function to set the reader chunk size to automatically get one partition per GPU\n", + "chunksize = dask_cugraph.get_chunksize(input_data_path)\n", + "\n", + "# Multi-GPU CSV reader\n", + "e_list = dask_cudf.read_csv(input_data_path,\n", + " chunksize = chunksize,\n", + " delimiter=' ',\n", + " names=['src', 'dst'],\n", + " dtype=['int32', 'int32'])\n", + "\n", + "G = cugraph.DiGraph()\n", + "G.from_dask_cudf_edgelist(e_list, source='src', destination='dst')\n", + "\n", + "# now run PageRank\n", + "pr_df = dask_cugraph.bfs(G, 1, depth_limit=10)\n", + "\n", + "# All done, clean up\n", + "comms.destroy()\n", + "client.close()\n", + "cluster.close()\n", + "\n", + "pr_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "comms.destroy()\n", + "client.close()\n", + "cluster.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "interpreter": { + "hash": "f708a36acfaef0acf74ccd43dfb58100269bf08fb79032a1e0a6f35bd9856f51" + }, + "kernelspec": { + "display_name": "Python 3.8.10 ('base')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/testing/testing1.py b/testing/testing1.py new file mode 100644 index 00000000000..774b861e3dc --- /dev/null +++ b/testing/testing1.py @@ -0,0 +1,37 @@ +# %% +from dask.distributed import Client, wait +from dask_cuda import LocalCUDACluster +import cugraph.dask.comms as Comms +import cugraph.dask as dask_cugraph + +input_data_path = '../datasets/karate.csv' + +cluster = LocalCUDACluster() +client = Client(cluster) +Comms.initialize(p2p=True) + +# Helper function to set the reader chunk size to automatically get one partition per GPU +chunksize = dask_cugraph.get_chunksize(input_data_path) + +# Multi-GPU CSV reader +e_list = dask_cudf.read_csv(input_data_path, + chunksize = chunksize, + delimiter=' ', + names=['src', 'dst'], + dtype=['int32', 'int32']) + +G = cugraph.DiGraph() +G.from_dask_cudf_edgelist(e_list, source='src', destination='dst') + +# now run PageRank +pr_df = dask_cugraph.pagerank(G, tol=1e-4) + +# All done, clean up +Comms.destroy() +client.close() +cluster.close() + +# %% + + + From af0eb2ae90a072cffbdad5d26703f72e4e105b55 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Wed, 18 May 2022 20:22:29 +0000 Subject: [PATCH 04/14] remove testing files --- testing/cufile.log | 480 ------------------ testing/dask-worker-space/global.lock | 0 testing/dask-worker-space/purge.lock | 0 .../dask-worker-space/worker-i66t8r65.dirlock | 0 .../dask-worker-space/worker-ifem89p4.dirlock | 0 .../dask-worker-space/worker-jvu0j__k.dirlock | 0 .../dask-worker-space/worker-l968y9vr.dirlock | 0 .../dask-worker-space/worker-lr81p2a2.dirlock | 0 .../dask-worker-space/worker-wjcu9c0j.dirlock | 0 .../dask-worker-space/worker-xtysqw6a.dirlock | 0 .../dask-worker-space/worker-ykczxdz3.dirlock | 0 testing/testing1.ipynb | 125 ----- testing/testing1.py | 37 -- 13 files changed, 642 deletions(-) delete mode 100644 testing/cufile.log delete mode 100644 testing/dask-worker-space/global.lock delete mode 100644 testing/dask-worker-space/purge.lock delete mode 100644 testing/dask-worker-space/worker-i66t8r65.dirlock delete mode 100644 testing/dask-worker-space/worker-ifem89p4.dirlock delete mode 100644 testing/dask-worker-space/worker-jvu0j__k.dirlock delete mode 100644 testing/dask-worker-space/worker-l968y9vr.dirlock delete mode 100644 testing/dask-worker-space/worker-lr81p2a2.dirlock delete mode 100644 testing/dask-worker-space/worker-wjcu9c0j.dirlock delete mode 100644 testing/dask-worker-space/worker-xtysqw6a.dirlock delete mode 100644 testing/dask-worker-space/worker-ykczxdz3.dirlock delete mode 100644 testing/testing1.ipynb delete mode 100644 testing/testing1.py diff --git a/testing/cufile.log b/testing/cufile.log deleted file mode 100644 index 8bea5444044..00000000000 --- a/testing/cufile.log +++ /dev/null @@ -1,480 +0,0 @@ - 17-05-2022 21:40:47:52 [pid=80962 tid=81356] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:40:47:53 [pid=80976 tid=81358] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:40:47:53 [pid=80967 tid=81357] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:40:47:54 [pid=80971 tid=81359] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:40:47:55 [pid=80968 tid=81360] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:40:47:55 [pid=80958 tid=81362] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:40:47:55 [pid=80979 tid=81361] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:40:47:55 [pid=80969 tid=81363] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:41:37:200 [pid=81717 tid=432] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:41:37:204 [pid=81693 tid=433] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:41:37:205 [pid=81710 tid=434] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:41:37:206 [pid=81719 tid=435] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:41:37:206 [pid=81708 tid=439] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:41:37:206 [pid=81699 tid=438] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:41:37:207 [pid=81703 tid=436] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:41:37:207 [pid=81686 tid=437] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:42:39:122 [pid=884 tid=1228] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:42:39:123 [pid=872 tid=1230] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:42:39:123 [pid=877 tid=1231] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:42:39:123 [pid=893 tid=1233] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:42:39:123 [pid=880 tid=1229] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:42:39:124 [pid=890 tid=1234] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:42:39:124 [pid=888 tid=1232] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:42:39:125 [pid=870 tid=1235] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:57:55:907 [pid=9582 tid=9944] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:57:55:908 [pid=9588 tid=9945] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:57:55:909 [pid=9576 tid=9946] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:57:55:909 [pid=9585 tid=9948] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:57:55:909 [pid=9579 tid=9947] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:57:55:909 [pid=9569 tid=9949] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:57:55:910 [pid=9565 tid=9950] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:57:55:910 [pid=9574 tid=9951] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:01:31:982 [pid=11929 tid=12279] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:01:31:982 [pid=11920 tid=12278] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:01:31:984 [pid=11915 tid=12281] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:01:31:984 [pid=11924 tid=12280] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:01:31:984 [pid=11918 tid=12282] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:01:31:985 [pid=11911 tid=12284] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:01:31:985 [pid=11926 tid=12285] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:01:31:985 [pid=11906 tid=12283] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:03:36:564 [pid=13364 tid=13750] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:03:36:564 [pid=13358 tid=13751] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:03:36:566 [pid=13346 tid=13752] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:03:36:566 [pid=13354 tid=13754] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:03:36:567 [pid=13341 tid=13753] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:03:36:568 [pid=13361 tid=13755] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:03:36:568 [pid=13353 tid=13756] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:03:36:568 [pid=13349 tid=13757] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:22:38:109 [pid=765 tid=1124] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:22:38:109 [pid=759 tid=1126] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:22:38:109 [pid=776 tid=1127] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:22:38:109 [pid=770 tid=1125] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:22:38:109 [pid=754 tid=1128] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:22:38:110 [pid=767 tid=1129] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:22:38:111 [pid=761 tid=1131] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:22:38:111 [pid=773 tid=1130] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:37:22:984 [pid=9037 tid=9437] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:37:22:987 [pid=9044 tid=9438] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:37:22:988 [pid=9051 tid=9443] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:37:22:988 [pid=9056 tid=9439] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:37:22:988 [pid=9042 tid=9441] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:37:22:988 [pid=9059 tid=9440] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:37:22:989 [pid=9053 tid=9444] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:37:22:989 [pid=9048 tid=9442] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:44:43:819 [pid=14440 tid=14774] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:44:43:824 [pid=14438 tid=14775] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:44:43:824 [pid=14434 tid=14776] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:44:43:824 [pid=14421 tid=14777] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:44:43:825 [pid=14429 tid=14781] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:44:43:825 [pid=14424 tid=14779] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:44:43:826 [pid=14443 tid=14778] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:44:43:826 [pid=14431 tid=14780] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:50:46:340 [pid=18062 tid=18415] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:50:46:341 [pid=18076 tid=18416] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:50:46:342 [pid=18066 tid=18418] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:50:46:342 [pid=18070 tid=18417] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:50:46:342 [pid=18081 tid=18419] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:50:46:343 [pid=18071 tid=18420] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:50:46:344 [pid=18078 tid=18421] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:50:46:344 [pid=18084 tid=18422] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:54:24:705 [pid=20369 tid=20749] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:54:24:710 [pid=20366 tid=20750] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:54:24:711 [pid=20351 tid=20753] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:54:24:711 [pid=20346 tid=20752] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:54:24:711 [pid=20355 tid=20751] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:54:24:712 [pid=20358 tid=20754] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:54:24:712 [pid=20360 tid=20755] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:54:24:712 [pid=20363 tid=20756] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:57:14:210 [pid=22242 tid=22584] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:57:14:211 [pid=22238 tid=22586] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:57:14:211 [pid=22225 tid=22585] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:57:14:212 [pid=22233 tid=22589] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:57:14:213 [pid=22228 tid=22587] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:57:14:213 [pid=22236 tid=22588] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:57:14:214 [pid=22248 tid=22591] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 22:57:14:215 [pid=22245 tid=22590] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:00:41:835 [pid=24294 tid=24659] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:00:41:838 [pid=24304 tid=24660] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:00:41:839 [pid=24316 tid=24661] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:00:41:839 [pid=24297 tid=24662] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:00:41:840 [pid=24302 tid=24663] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:00:41:840 [pid=24313 tid=24664] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:00:41:841 [pid=24308 tid=24666] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:00:41:841 [pid=24309 tid=24665] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:02:48:381 [pid=741 tid=1079] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:02:48:382 [pid=736 tid=1080] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:02:48:382 [pid=752 tid=1081] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:02:48:382 [pid=733 tid=1084] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:02:48:382 [pid=749 tid=1083] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:02:48:382 [pid=746 tid=1086] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:02:48:382 [pid=743 tid=1082] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:02:48:383 [pid=755 tid=1085] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:04:46:755 [pid=2219 tid=2562] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:04:46:755 [pid=2213 tid=2561] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:04:46:759 [pid=2209 tid=2565] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:04:46:760 [pid=2222 tid=2563] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:04:46:760 [pid=2202 tid=2564] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:04:46:760 [pid=2206 tid=2566] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:04:46:760 [pid=2224 tid=2567] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:04:46:762 [pid=2216 tid=2568] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:12:29:196 [pid=6696 tid=7058] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:12:29:197 [pid=6709 tid=7065] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:12:29:197 [pid=6687 tid=7060] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:12:29:197 [pid=6697 tid=7059] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:12:29:197 [pid=6704 tid=7061] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:12:29:198 [pid=6682 tid=7062] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:12:29:198 [pid=6706 tid=7064] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:12:29:198 [pid=6701 tid=7063] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:22:31:944 [pid=12411 tid=12763] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:22:31:945 [pid=12409 tid=12764] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:22:31:946 [pid=12393 tid=12766] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:22:31:946 [pid=12389 tid=12765] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:22:31:947 [pid=12405 tid=12767] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:22:31:947 [pid=12403 tid=12768] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:22:31:948 [pid=12398 tid=12770] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:22:31:948 [pid=12396 tid=12769] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:25:42:8 [pid=14490 tid=14865] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:25:42:8 [pid=14502 tid=14864] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:25:42:8 [pid=14492 tid=14863] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:25:42:9 [pid=14498 tid=14867] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:25:42:9 [pid=14504 tid=14866] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:25:42:9 [pid=14495 tid=14868] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:25:42:9 [pid=14482 tid=14870] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:25:42:9 [pid=14486 tid=14869] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:42:34:810 [pid=23802 tid=24177] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:42:34:810 [pid=23791 tid=24180] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:42:34:810 [pid=23808 tid=24181] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:42:34:810 [pid=23798 tid=24183] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:42:34:810 [pid=23811 tid=24182] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:42:34:810 [pid=23805 tid=24178] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:42:34:810 [pid=23794 tid=24176] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:42:34:811 [pid=23813 tid=24179] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:46:58:752 [pid=26563 tid=26929] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:46:58:754 [pid=26574 tid=26931] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:46:58:754 [pid=26572 tid=26933] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:46:58:755 [pid=26581 tid=26932] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:46:58:755 [pid=26577 tid=26930] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:46:58:755 [pid=26583 tid=26935] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:46:58:755 [pid=26566 tid=26934] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:46:58:757 [pid=26584 tid=26936] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:50:12:847 [pid=28661 tid=29053] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:50:12:850 [pid=28670 tid=29054] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:50:12:851 [pid=28668 tid=29055] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:50:12:851 [pid=28664 tid=29056] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:50:12:851 [pid=28655 tid=29057] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:50:12:852 [pid=28658 tid=29058] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:50:12:853 [pid=28651 tid=29059] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:50:12:853 [pid=28648 tid=29060] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:55:42:202 [pid=32311 tid=32708] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:55:42:202 [pid=32297 tid=32706] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:55:42:202 [pid=32316 tid=32707] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:55:42:203 [pid=32302 tid=32709] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:55:42:204 [pid=32294 tid=32712] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:55:42:204 [pid=32313 tid=32710] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:55:42:204 [pid=32305 tid=32711] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 23:55:42:204 [pid=32308 tid=32713] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:02:42:130 [pid=36342 tid=36777] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:02:42:132 [pid=36337 tid=36778] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:02:42:132 [pid=36352 tid=36779] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:02:42:133 [pid=36346 tid=36780] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:02:42:133 [pid=36359 tid=36781] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:02:42:133 [pid=36354 tid=36782] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:02:42:134 [pid=36363 tid=36783] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:02:42:134 [pid=36348 tid=36784] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:11:54:868 [pid=42479 tid=42838] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:11:54:868 [pid=42469 tid=42839] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:11:54:869 [pid=42490 tid=42841] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:11:54:869 [pid=42475 tid=42840] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:11:54:871 [pid=42480 tid=42844] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:11:54:871 [pid=42485 tid=42842] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:11:54:871 [pid=42481 tid=42843] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:11:54:871 [pid=42489 tid=42845] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:14:18:596 [pid=44179 tid=44528] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:14:18:601 [pid=44175 tid=44530] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:14:18:601 [pid=44190 tid=44531] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:14:18:602 [pid=44171 tid=44533] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:14:18:602 [pid=44187 tid=44532] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:14:18:602 [pid=44185 tid=44534] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:14:18:602 [pid=44181 tid=44529] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:14:18:602 [pid=44193 tid=44535] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:17:58:669 [pid=46538 tid=46935] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:17:58:671 [pid=46547 tid=46936] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:17:58:672 [pid=46560 tid=46938] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:17:58:672 [pid=46533 tid=46940] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:17:58:672 [pid=46554 tid=46937] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:17:58:672 [pid=46541 tid=46939] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:17:58:672 [pid=46544 tid=46941] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:17:58:673 [pid=46550 tid=46942] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:23:04:395 [pid=49642 tid=50057] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:23:04:397 [pid=49622 tid=50059] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:23:04:397 [pid=49623 tid=50058] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:23:04:397 [pid=49636 tid=50061] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:23:04:397 [pid=49631 tid=50062] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:23:04:397 [pid=49639 tid=50060] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:23:04:398 [pid=49616 tid=50063] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:23:04:399 [pid=49613 tid=50064] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:25:02:949 [pid=51073 tid=51497] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:25:02:951 [pid=51085 tid=51498] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:25:02:951 [pid=51067 tid=51499] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:25:02:951 [pid=51063 tid=51501] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:25:02:951 [pid=51082 tid=51500] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:25:02:951 [pid=51072 tid=51503] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:25:02:952 [pid=51077 tid=51502] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:25:02:953 [pid=51079 tid=51504] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:28:15:196 [pid=53385 tid=53810] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:28:15:196 [pid=53392 tid=53811] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:28:15:198 [pid=53398 tid=53812] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:28:15:198 [pid=53396 tid=53816] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:28:15:198 [pid=53384 tid=53813] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:28:15:198 [pid=53377 tid=53814] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:28:15:198 [pid=53379 tid=53815] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 00:28:15:199 [pid=53372 tid=53817] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:44:28:218 [pid=77209 tid=77611] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:44:28:219 [pid=77214 tid=77612] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:44:28:220 [pid=77221 tid=77613] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:44:28:221 [pid=77217 tid=77614] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:44:28:221 [pid=77205 tid=77615] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:44:28:221 [pid=77212 tid=77616] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:44:28:222 [pid=77223 tid=77617] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:44:28:222 [pid=77201 tid=77618] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:48:05:448 [pid=79550 tid=79941] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:48:05:448 [pid=79545 tid=79940] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:48:05:450 [pid=79535 tid=79942] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:48:05:450 [pid=79553 tid=79943] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:48:05:451 [pid=79531 tid=79944] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:48:05:451 [pid=79547 tid=79945] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:48:05:452 [pid=79539 tid=79946] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:48:05:454 [pid=79542 tid=79947] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:52:05:661 [pid=334 tid=772] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:52:05:664 [pid=355 tid=773] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:52:05:665 [pid=352 tid=775] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:52:05:665 [pid=338 tid=774] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:52:05:666 [pid=358 tid=776] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:52:05:666 [pid=350 tid=777] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:52:05:667 [pid=340 tid=778] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:52:05:667 [pid=347 tid=779] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:56:35:733 [pid=2851 tid=3285] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:56:35:733 [pid=2857 tid=3286] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:56:35:733 [pid=2867 tid=3287] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:56:35:734 [pid=2871 tid=3288] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:56:35:735 [pid=2865 tid=3292] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:56:35:735 [pid=2855 tid=3291] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:56:35:737 [pid=2862 tid=3289] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 02:56:35:737 [pid=2873 tid=3290] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:02:42:426 [pid=6301 tid=6731] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:02:42:427 [pid=6290 tid=6732] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:02:42:428 [pid=6304 tid=6733] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:02:42:428 [pid=6286 tid=6734] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:02:42:429 [pid=6298 tid=6736] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:02:42:429 [pid=6282 tid=6735] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:02:42:429 [pid=6294 tid=6737] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:02:42:430 [pid=6293 tid=6738] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:05:20:738 [pid=8267 tid=8606] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:05:20:741 [pid=8261 tid=8609] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:05:20:741 [pid=8270 tid=8607] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:05:20:741 [pid=8277 tid=8611] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:05:20:742 [pid=8284 tid=8608] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:05:20:742 [pid=8257 tid=8610] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:05:20:742 [pid=8265 tid=8612] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:05:20:743 [pid=8280 tid=8613] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:10:43:600 [pid=18195 tid=18568] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:10:43:600 [pid=18205 tid=18572] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:10:43:601 [pid=18202 tid=18569] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:10:43:601 [pid=18185 tid=18571] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:10:43:601 [pid=18201 tid=18570] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:10:43:601 [pid=18183 tid=18573] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:10:43:603 [pid=18196 tid=18574] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:10:43:603 [pid=18191 tid=18575] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:14:15:563 [pid=20251 tid=20510] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:14:15:716 [pid=20244 tid=20514] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:14:15:738 [pid=20237 tid=20515] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:14:15:772 [pid=20256 tid=20516] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:14:15:790 [pid=20234 tid=20517] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:14:15:812 [pid=20248 tid=20518] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:14:15:824 [pid=20259 tid=20519] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:14:15:834 [pid=20231 tid=20520] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:19:32:698 [pid=23570 tid=23968] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:19:32:700 [pid=23578 tid=23970] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:19:32:701 [pid=23559 tid=23972] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:19:32:701 [pid=23575 tid=23973] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:19:32:701 [pid=23565 tid=23971] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:19:32:702 [pid=23553 tid=23969] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:19:32:703 [pid=23557 tid=23974] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:19:32:704 [pid=23564 tid=23975] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:22:14:798 [pid=25386 tid=25816] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:22:14:801 [pid=25393 tid=25818] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:22:14:801 [pid=25399 tid=25819] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:22:14:801 [pid=25388 tid=25817] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:22:14:801 [pid=25396 tid=25821] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:22:14:802 [pid=25407 tid=25820] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:22:14:802 [pid=25404 tid=25822] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:22:14:803 [pid=25402 tid=25823] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:25:02:610 [pid=27235 tid=27638] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:25:02:612 [pid=27210 tid=27639] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:25:02:613 [pid=27223 tid=27640] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:25:02:614 [pid=27227 tid=27641] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:25:02:614 [pid=27231 tid=27642] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:25:02:614 [pid=27220 tid=27645] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:25:02:614 [pid=27213 tid=27643] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:25:02:614 [pid=27217 tid=27644] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:28:05:236 [pid=29145 tid=29593] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:28:05:240 [pid=29148 tid=29594] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:28:05:241 [pid=29153 tid=29595] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:28:05:241 [pid=29156 tid=29596] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:28:05:241 [pid=29150 tid=29597] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:28:05:241 [pid=29134 tid=29598] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:28:05:242 [pid=29138 tid=29600] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:28:05:242 [pid=29142 tid=29599] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:32:43:258 [pid=32655 tid=32989] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:32:43:258 [pid=32652 tid=32990] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:32:43:259 [pid=32649 tid=32991] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:32:43:260 [pid=32642 tid=32992] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:32:43:260 [pid=32657 tid=32995] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:32:43:260 [pid=32638 tid=32993] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:32:43:260 [pid=32645 tid=32994] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:32:43:261 [pid=32661 tid=32996] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:34:41:896 [pid=34038 tid=34430] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:34:41:896 [pid=34043 tid=34428] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:34:41:897 [pid=34046 tid=34429] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:34:41:898 [pid=34029 tid=34431] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:34:41:898 [pid=34040 tid=34432] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:34:41:898 [pid=34032 tid=34433] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:34:41:898 [pid=34025 tid=34434] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:34:41:899 [pid=34021 tid=34435] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:36:41:608 [pid=35324 tid=35724] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:36:41:613 [pid=35330 tid=35728] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:36:41:613 [pid=35348 tid=35725] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:36:41:613 [pid=35342 tid=35726] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:36:41:614 [pid=35346 tid=35727] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:36:41:614 [pid=35338 tid=35729] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:36:41:614 [pid=35337 tid=35730] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:36:41:615 [pid=35334 tid=35731] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:38:47:542 [pid=36799 tid=37215] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:38:47:543 [pid=36814 tid=37216] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:38:47:543 [pid=36811 tid=37214] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:38:47:543 [pid=36817 tid=37218] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:38:47:543 [pid=36802 tid=37217] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:38:47:543 [pid=36805 tid=37219] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:38:47:544 [pid=36808 tid=37220] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:38:47:545 [pid=36795 tid=37221] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:44:33:312 [pid=40804 tid=41135] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:44:33:312 [pid=40781 tid=41137] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:44:33:312 [pid=40808 tid=41136] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:44:33:313 [pid=40801 tid=41138] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:44:33:314 [pid=40789 tid=41139] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:44:33:314 [pid=40798 tid=41140] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:44:33:314 [pid=40792 tid=41141] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 03:44:33:316 [pid=40795 tid=41142] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:55:13:531 [pid=71995 tid=72369] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:55:13:531 [pid=71999 tid=72373] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:55:13:531 [pid=72011 tid=72368] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:55:13:531 [pid=72005 tid=72372] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:55:13:531 [pid=72007 tid=72370] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:55:13:531 [pid=71990 tid=72375] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:55:13:531 [pid=72001 tid=72374] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:55:13:531 [pid=72013 tid=72371] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:56:58:187 [pid=73488 tid=73827] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:56:58:187 [pid=73471 tid=73828] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:56:58:188 [pid=73477 tid=73830] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:56:58:188 [pid=73498 tid=73829] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:56:58:189 [pid=73481 tid=73831] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:56:58:189 [pid=73472 tid=73832] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:56:58:189 [pid=73492 tid=73833] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:56:58:189 [pid=73483 tid=73834] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:59:44:910 [pid=75350 tid=75690] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:59:44:913 [pid=75354 tid=75691] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:59:44:913 [pid=75347 tid=75692] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:59:44:914 [pid=75342 tid=75696] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:59:44:915 [pid=75345 tid=75693] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:59:44:916 [pid=75334 tid=75695] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:59:44:916 [pid=75337 tid=75694] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 12:59:44:917 [pid=75356 tid=75697] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:02:42:583 [pid=77681 tid=78034] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:02:42:583 [pid=77692 tid=78036] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:02:42:583 [pid=77691 tid=78035] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:02:42:584 [pid=77699 tid=78038] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:02:42:584 [pid=77684 tid=78039] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:02:42:584 [pid=77676 tid=78037] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:02:42:584 [pid=77688 tid=78040] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:02:42:585 [pid=77696 tid=78041] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:05:04:504 [pid=79223 tid=79576] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:05:04:507 [pid=79218 tid=79577] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:05:04:509 [pid=79232 tid=79578] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:05:04:509 [pid=79210 tid=79579] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:05:04:509 [pid=79214 tid=79582] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:05:04:509 [pid=79226 tid=79581] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:05:04:510 [pid=79219 tid=79580] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:05:04:510 [pid=79229 tid=79583] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:07:52:128 [pid=81383 tid=81774] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:07:52:128 [pid=81406 tid=81778] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:07:52:129 [pid=81400 tid=81775] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:07:52:129 [pid=81403 tid=81776] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:07:52:129 [pid=81398 tid=81777] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:07:52:129 [pid=81395 tid=81779] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:07:52:131 [pid=81389 tid=81780] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:07:52:131 [pid=81391 tid=81781] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:27:20:252 [pid=10465 tid=10867] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:27:20:253 [pid=10466 tid=10870] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:27:20:253 [pid=10454 tid=10869] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:27:20:253 [pid=10459 tid=10868] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:27:20:253 [pid=10471 tid=10871] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:27:20:255 [pid=10477 tid=10872] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:27:20:255 [pid=10462 tid=10874] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:27:20:255 [pid=10480 tid=10873] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:35:11:429 [pid=15345 tid=15704] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:35:11:429 [pid=15358 tid=15705] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:35:11:429 [pid=15361 tid=15709] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:35:11:429 [pid=15352 tid=15706] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:35:11:429 [pid=15348 tid=15710] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:35:11:429 [pid=15341 tid=15707] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:35:11:430 [pid=15354 tid=15708] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 13:35:11:430 [pid=15363 tid=15711] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 14:08:16:795 [pid=28767 tid=29148] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 14:08:16:795 [pid=28758 tid=29150] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 14:08:16:795 [pid=28778 tid=29152] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 14:08:16:795 [pid=28773 tid=29154] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 14:08:16:795 [pid=28774 tid=29153] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 14:08:16:795 [pid=28762 tid=29151] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 14:08:16:795 [pid=28780 tid=29149] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 14:08:16:795 [pid=28768 tid=29155] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 14:14:40:101 [pid=32624 tid=33038] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 14:14:40:101 [pid=32608 tid=33036] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 14:14:40:102 [pid=32622 tid=33037] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 14:14:40:102 [pid=32602 tid=33039] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 14:14:40:103 [pid=32613 tid=33040] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 14:14:40:103 [pid=32616 tid=33043] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 14:14:40:103 [pid=32619 tid=33041] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 14:14:40:103 [pid=32606 tid=33042] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 15:58:20:951 [pid=21836 tid=22705] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 15:58:20:952 [pid=21847 tid=22704] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 15:58:20:952 [pid=21841 tid=22703] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 15:58:20:952 [pid=21838 tid=22707] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 15:58:20:952 [pid=21844 tid=22706] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 15:58:20:952 [pid=21825 tid=22708] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 15:58:20:952 [pid=21829 tid=22709] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 15:58:20:953 [pid=21833 tid=22710] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:07:52:420 [pid=13057 tid=13482] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:07:52:422 [pid=13045 tid=13484] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:07:52:422 [pid=13041 tid=13483] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:07:52:423 [pid=13049 tid=13487] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:07:52:423 [pid=13060 tid=13486] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:07:52:424 [pid=13064 tid=13488] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:07:52:424 [pid=13054 tid=13489] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:07:52:425 [pid=13051 tid=13485] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:09:38:528 [pid=14499 tid=14923] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:09:38:529 [pid=14496 tid=14922] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:09:38:529 [pid=14489 tid=14924] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:09:38:530 [pid=14492 tid=14925] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:09:38:530 [pid=14491 tid=14927] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:09:38:530 [pid=14477 tid=14926] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:09:38:531 [pid=14502 tid=14928] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:09:38:531 [pid=14485 tid=14929] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:10:33:230 [pid=15322 tid=15762] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:10:33:231 [pid=15334 tid=15763] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:10:33:232 [pid=15317 tid=15765] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:10:33:233 [pid=15326 tid=15766] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:10:33:233 [pid=15331 tid=15767] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:10:33:233 [pid=15328 tid=15764] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:10:33:235 [pid=15312 tid=15768] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:10:33:235 [pid=15319 tid=15769] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded diff --git a/testing/dask-worker-space/global.lock b/testing/dask-worker-space/global.lock deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testing/dask-worker-space/purge.lock b/testing/dask-worker-space/purge.lock deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testing/dask-worker-space/worker-i66t8r65.dirlock b/testing/dask-worker-space/worker-i66t8r65.dirlock deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testing/dask-worker-space/worker-ifem89p4.dirlock b/testing/dask-worker-space/worker-ifem89p4.dirlock deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testing/dask-worker-space/worker-jvu0j__k.dirlock b/testing/dask-worker-space/worker-jvu0j__k.dirlock deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testing/dask-worker-space/worker-l968y9vr.dirlock b/testing/dask-worker-space/worker-l968y9vr.dirlock deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testing/dask-worker-space/worker-lr81p2a2.dirlock b/testing/dask-worker-space/worker-lr81p2a2.dirlock deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testing/dask-worker-space/worker-wjcu9c0j.dirlock b/testing/dask-worker-space/worker-wjcu9c0j.dirlock deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testing/dask-worker-space/worker-xtysqw6a.dirlock b/testing/dask-worker-space/worker-xtysqw6a.dirlock deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testing/dask-worker-space/worker-ykczxdz3.dirlock b/testing/dask-worker-space/worker-ykczxdz3.dirlock deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testing/testing1.ipynb b/testing/testing1.ipynb deleted file mode 100644 index 07b980c5c24..00000000000 --- a/testing/testing1.ipynb +++ /dev/null @@ -1,125 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2022-05-18 20:09:21,992 - distributed.diskutils - INFO - Found stale lock file and directory '/root/cugraph/testing/dask-worker-space/worker-14xf1dwk', purging\n", - "2022-05-18 20:09:21,992 - distributed.diskutils - INFO - Found stale lock file and directory '/root/cugraph/testing/dask-worker-space/worker-14_bx41d', purging\n", - "2022-05-18 20:09:21,993 - distributed.diskutils - INFO - Found stale lock file and directory '/root/cugraph/testing/dask-worker-space/worker-phli1i9q', purging\n", - "2022-05-18 20:09:21,993 - distributed.diskutils - INFO - Found stale lock file and directory '/root/cugraph/testing/dask-worker-space/worker-ekqa9enz', purging\n", - "2022-05-18 20:09:21,993 - distributed.diskutils - INFO - Found stale lock file and directory '/root/cugraph/testing/dask-worker-space/worker-y77x0ytl', purging\n", - "2022-05-18 20:09:21,993 - distributed.diskutils - INFO - Found stale lock file and directory '/root/cugraph/testing/dask-worker-space/worker-807hal14', purging\n", - "2022-05-18 20:09:21,994 - distributed.diskutils - INFO - Found stale lock file and directory '/root/cugraph/testing/dask-worker-space/worker-sv410h22', purging\n", - "2022-05-18 20:09:21,994 - distributed.diskutils - INFO - Found stale lock file and directory '/root/cugraph/testing/dask-worker-space/worker-myjjhguf', purging\n", - "2022-05-18 20:09:21,994 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", - "2022-05-18 20:09:22,025 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", - "2022-05-18 20:09:22,039 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", - "2022-05-18 20:09:22,044 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", - "2022-05-18 20:09:22,050 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", - "2022-05-18 20:09:22,051 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", - "2022-05-18 20:09:22,054 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", - "2022-05-18 20:09:22,064 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", - "/opt/conda/envs/rapids/lib/python3.8/site-packages/cudf/core/dataframe.py:1055: FutureWarning: The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning.\n", - " return pd.Series(self._dtypes)\n" - ] - }, - { - "ename": "TypeError", - "evalue": "bfs() got an unexpected keyword argument 'do_expensive_check'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/tmp/ipykernel_14334/244951834.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 25\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[0;31m# now run PageRank\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m \u001b[0mpr_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdask_cugraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbfs\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mG\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth_limit\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdo_expensive_check\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0;31m# All done, clean up\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mTypeError\u001b[0m: bfs() got an unexpected keyword argument 'do_expensive_check'" - ] - } - ], - "source": [ - "from dask.distributed import Client, wait\n", - "from dask_cuda import LocalCUDACluster\n", - "from cugraph.dask.comms import comms\n", - "import cugraph.dask as dask_cugraph\n", - "import cugraph, dask_cudf\n", - "\n", - "input_data_path = '../datasets/karate.csv'\n", - "\n", - "cluster = LocalCUDACluster()\n", - "client = Client(cluster)\n", - "comms.initialize(p2p=True)\n", - "\n", - "# Helper function to set the reader chunk size to automatically get one partition per GPU\n", - "chunksize = dask_cugraph.get_chunksize(input_data_path)\n", - "\n", - "# Multi-GPU CSV reader\n", - "e_list = dask_cudf.read_csv(input_data_path,\n", - " chunksize = chunksize,\n", - " delimiter=' ',\n", - " names=['src', 'dst'],\n", - " dtype=['int32', 'int32'])\n", - "\n", - "G = cugraph.DiGraph()\n", - "G.from_dask_cudf_edgelist(e_list, source='src', destination='dst')\n", - "\n", - "# now run PageRank\n", - "pr_df = dask_cugraph.bfs(G, 1, depth_limit=10)\n", - "\n", - "# All done, clean up\n", - "comms.destroy()\n", - "client.close()\n", - "cluster.close()\n", - "\n", - "pr_df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "comms.destroy()\n", - "client.close()\n", - "cluster.close()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "interpreter": { - "hash": "f708a36acfaef0acf74ccd43dfb58100269bf08fb79032a1e0a6f35bd9856f51" - }, - "kernelspec": { - "display_name": "Python 3.8.10 ('base')", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/testing/testing1.py b/testing/testing1.py deleted file mode 100644 index 774b861e3dc..00000000000 --- a/testing/testing1.py +++ /dev/null @@ -1,37 +0,0 @@ -# %% -from dask.distributed import Client, wait -from dask_cuda import LocalCUDACluster -import cugraph.dask.comms as Comms -import cugraph.dask as dask_cugraph - -input_data_path = '../datasets/karate.csv' - -cluster = LocalCUDACluster() -client = Client(cluster) -Comms.initialize(p2p=True) - -# Helper function to set the reader chunk size to automatically get one partition per GPU -chunksize = dask_cugraph.get_chunksize(input_data_path) - -# Multi-GPU CSV reader -e_list = dask_cudf.read_csv(input_data_path, - chunksize = chunksize, - delimiter=' ', - names=['src', 'dst'], - dtype=['int32', 'int32']) - -G = cugraph.DiGraph() -G.from_dask_cudf_edgelist(e_list, source='src', destination='dst') - -# now run PageRank -pr_df = dask_cugraph.pagerank(G, tol=1e-4) - -# All done, clean up -Comms.destroy() -client.close() -cluster.close() - -# %% - - - From 97b76e716c46d6e0f479b26cc547b05a9965c0fc Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Thu, 19 May 2022 19:18:57 +0000 Subject: [PATCH 05/14] update mg bfs test --- cpp/tests/c_api/mg_bfs_test.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/tests/c_api/mg_bfs_test.c b/cpp/tests/c_api/mg_bfs_test.c index d1e303fffb1..2975808cd75 100644 --- a/cpp/tests/c_api/mg_bfs_test.c +++ b/cpp/tests/c_api/mg_bfs_test.c @@ -72,6 +72,8 @@ int generic_bfs_test( TRUE, &paths_result, &ret_error); + + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "cugraph_bfs failed."); cugraph_type_erased_device_array_view_t* vertices; From 3b9914f83e59b9965d655046346aa28169abc34d Mon Sep 17 00:00:00 2001 From: Chuck Hastings Date: Thu, 19 May 2022 15:56:40 -0400 Subject: [PATCH 06/14] Add Alex B's mg_bfs_test, add mg_sssp_test, fix support for MG in bfs and sssp --- cpp/src/c_api/bfs.cpp | 19 +++- cpp/src/c_api/sssp.cpp | 4 +- cpp/tests/CMakeLists.txt | 2 + cpp/tests/c_api/mg_bfs_test.c | 180 +++++++++++++++++++++++++++++++ cpp/tests/c_api/mg_sssp_test.c | 188 +++++++++++++++++++++++++++++++++ 5 files changed, 386 insertions(+), 7 deletions(-) create mode 100644 cpp/tests/c_api/mg_bfs_test.c create mode 100644 cpp/tests/c_api/mg_sssp_test.c diff --git a/cpp/src/c_api/bfs.cpp b/cpp/src/c_api/bfs.cpp index 3e2367be423..dcfadf5944f 100644 --- a/cpp/src/c_api/bfs.cpp +++ b/cpp/src/c_api/bfs.cpp @@ -23,6 +23,7 @@ #include #include +#include #include #include @@ -89,15 +90,23 @@ struct bfs_functor : public abstract_functor { rmm::device_uvector predecessors(0, handle_.get_stream()); if (compute_predecessors_) { - predecessors.resize(graph->number_of_vertices(), handle_.get_stream()); + predecessors.resize(graph_view.local_vertex_partition_range_size(), handle_.get_stream()); + } + + rmm::device_uvector sources(sources_->size_, handle_.get_stream()); + raft::copy( + sources.data(), sources_->as_type(), sources_->size_, handle_.get_stream()); + + if constexpr (multi_gpu) { + sources = detail::shuffle_ext_vertices_by_gpu_id(handle_, std::move(sources)); } // // Need to renumber sources // renumber_ext_vertices(handle_, - sources_->as_type(), - sources_->size_, + sources.data(), + sources.size(), number_map->data(), graph_view.local_vertex_partition_range_first(), graph_view.local_vertex_partition_range_last(), @@ -108,8 +117,8 @@ struct bfs_functor : public abstract_functor { graph_view, distances.data(), compute_predecessors_ ? predecessors.data() : nullptr, - sources_->as_type(), - sources_->size_, + sources.data(), + sources.size(), direction_optimizing_, static_cast(depth_limit_), do_expensive_check_); diff --git a/cpp/src/c_api/sssp.cpp b/cpp/src/c_api/sssp.cpp index 24ef5f751e2..5e494f69d2b 100644 --- a/cpp/src/c_api/sssp.cpp +++ b/cpp/src/c_api/sssp.cpp @@ -87,7 +87,7 @@ struct sssp_functor : public abstract_functor { rmm::device_uvector predecessors(0, handle_.get_stream()); if (compute_predecessors_) { - predecessors.resize(graph->number_of_vertices(), handle_.get_stream()); + predecessors.resize(graph_view.local_vertex_partition_range_size(), handle_.get_stream()); } vertex_t src = static_cast(source_); @@ -98,7 +98,7 @@ struct sssp_functor : public abstract_functor { // renumber_ext_vertices(handle_, source_ids.data(), - 1, + source_ids.size(), number_map->data(), graph_view.local_vertex_partition_range_first(), graph_view.local_vertex_partition_range_last(), diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index cab8966c0f6..15ff56acfb6 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -636,6 +636,8 @@ if(BUILD_CUGRAPH_MG_TESTS) # - MG C API tests ------------------------------------------------------------------------ ConfigureCTestMG(MG_CAPI_CREATE_GRAPH c_api/mg_create_graph_test.c c_api/mg_test_utils.cpp) ConfigureCTestMG(MG_CAPI_PAGERANK c_api/mg_pagerank_test.c c_api/mg_test_utils.cpp) + ConfigureCTestMG(MG_CAPI_BFS c_api/mg_bfs_test.c c_api/mg_test_utils.cpp) + ConfigureCTestMG(MG_CAPI_SSSP c_api/mg_sssp_test.c c_api/mg_test_utils.cpp) ConfigureCTestMG(MG_CAPI_WEAKLY_CONNECTED_COMPONENTS c_api/mg_weakly_connected_components_test.c c_api/mg_test_utils.cpp) ConfigureCTestMG(MG_CAPI_STRONGLY_CONNECTED_COMPONENTS c_api/mg_strongly_connected_components_test.c c_api/mg_test_utils.cpp) ConfigureCTestMG(MG_CAPI_KATZ c_api/mg_katz_test.c c_api/mg_test_utils.cpp) diff --git a/cpp/tests/c_api/mg_bfs_test.c b/cpp/tests/c_api/mg_bfs_test.c new file mode 100644 index 00000000000..0fc1fc431b5 --- /dev/null +++ b/cpp/tests/c_api/mg_bfs_test.c @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2022, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mg_test_utils.h" /* RUN_TEST */ + +#include +#include + +#include + +typedef int32_t vertex_t; +typedef int32_t edge_t; +typedef float weight_t; + +int generic_bfs_test( + const cugraph_resource_handle_t* p_handle, + vertex_t* h_src, + vertex_t* h_dst, + weight_t* h_wgt, + vertex_t* h_seeds, + vertex_t const* expected_distances, + vertex_t const* expected_predecessors, + size_t num_vertices, + size_t num_edges, + size_t num_seeds, + size_t depth_limit, + bool_t store_transposed) { + int test_ret_value = 0; + + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error; + + cugraph_graph_t* p_graph = NULL; + cugraph_paths_result_t* paths_result = NULL; + cugraph_type_erased_device_array_t* p_sources = NULL; + cugraph_type_erased_device_array_view_t* p_source_view = NULL; + + ret_code = + cugraph_type_erased_device_array_create(p_handle, num_seeds, INT32, &p_sources, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "p_sources create failed."); + + p_source_view = cugraph_type_erased_device_array_view(p_sources); + + ret_code = cugraph_type_erased_device_array_view_copy_from_host( + p_handle, p_source_view, (byte_t*)h_seeds, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "src copy_from_host failed."); + + ret_code = create_mg_test_graph( + p_handle, h_src, h_dst, h_wgt, num_edges, store_transposed, FALSE, &p_graph, &ret_error); + + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "create_mg_test_graph failed."); + + ret_code = cugraph_bfs(p_handle, + p_graph, + p_source_view, + FALSE, + 10000000, + TRUE, + TRUE, + &paths_result, + &ret_error); + + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "cugraph_bfs failed."); + + cugraph_type_erased_device_array_view_t* vertices; + cugraph_type_erased_device_array_view_t* distances; + cugraph_type_erased_device_array_view_t* predecessors; + + vertices = cugraph_paths_result_get_vertices(paths_result); + predecessors = cugraph_paths_result_get_predecessors(paths_result); + distances = cugraph_paths_result_get_distances(paths_result); + + vertex_t h_vertices[num_vertices]; + vertex_t h_predecessors[num_vertices]; + vertex_t h_distances[num_vertices]; + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + p_handle, (byte_t*)h_vertices, vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + p_handle, (byte_t*)h_distances, distances, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + p_handle, (byte_t*)h_predecessors, predecessors, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + size_t num_local_vertices = cugraph_type_erased_device_array_view_size(vertices); + + for (int i = 0; (i < num_local_vertices) && (test_ret_value == 0); ++i) { + TEST_ASSERT(test_ret_value, + expected_distances[h_vertices[i]] == h_distances[i], + "bfs distances don't match"); + + + TEST_ASSERT(test_ret_value, + expected_predecessors[h_vertices[i]] == h_predecessors[i], + "bfs predecessors don't match"); + } + + cugraph_paths_result_free(paths_result); + cugraph_mg_graph_free(p_graph); + cugraph_error_free(ret_error); + + return test_ret_value; +} + +int test_bfs(const cugraph_resource_handle_t* p_handle) +{ + size_t num_edges = 8; + size_t num_vertices = 6; + + vertex_t src[] = {0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t dst[] = {1, 3, 4, 0, 1, 3, 5, 5}; + weight_t wgt[] = {0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f}; + vertex_t seeds[] = {0}; + vertex_t expected_distances[] = {0, 1, 2147483647, 2, 2, 3}; + vertex_t expected_predecessors[] = {-1, 0, -1, 1, 1, 3}; + + // Bfs wants store_transposed = FALSE + return generic_bfs_test(p_handle, + src, + dst, + wgt, + seeds, + expected_distances, + expected_predecessors, + num_vertices, + num_edges, + 1, + 10, + FALSE); +} + +int main(int argc, char** argv) +{ + // Set up MPI: + int comm_rank; + int comm_size; + int num_gpus_per_node; + cudaError_t status; + int mpi_status; + cugraph_resource_handle_t* handle = NULL; + cugraph_error_t* ret_error; + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + int prows = 1; + + C_MPI_TRY(MPI_Init(&argc, &argv)); + C_MPI_TRY(MPI_Comm_rank(MPI_COMM_WORLD, &comm_rank)); + C_MPI_TRY(MPI_Comm_size(MPI_COMM_WORLD, &comm_size)); + C_CUDA_TRY(cudaGetDeviceCount(&num_gpus_per_node)); + C_CUDA_TRY(cudaSetDevice(comm_rank % num_gpus_per_node)); + + void* raft_handle = create_raft_handle(prows); + handle = cugraph_create_resource_handle(raft_handle); + int result = 0; + result |= RUN_MG_TEST(test_bfs, handle); + + cugraph_free_resource_handle(handle); + free_raft_handle(raft_handle); + + C_MPI_TRY(MPI_Finalize()); + + return result; +} diff --git a/cpp/tests/c_api/mg_sssp_test.c b/cpp/tests/c_api/mg_sssp_test.c new file mode 100644 index 00000000000..3aca77dca28 --- /dev/null +++ b/cpp/tests/c_api/mg_sssp_test.c @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2022, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mg_test_utils.h" /* RUN_TEST */ + +#include +#include + +#include +#include + +typedef int32_t vertex_t; +typedef int32_t edge_t; +typedef float weight_t; + +const weight_t EPSILON = 0.001; + +int generic_sssp_test(const cugraph_resource_handle_t* p_handle, + vertex_t* h_src, + vertex_t* h_dst, + weight_t* h_wgt, + vertex_t source, + weight_t const* expected_distances, + vertex_t const* expected_predecessors, + size_t num_vertices, + size_t num_edges, + weight_t cutoff, + bool_t store_transposed) +{ + int test_ret_value = 0; + + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + cugraph_error_t* ret_error; + + cugraph_graph_t* p_graph = NULL; + cugraph_paths_result_t* p_result = NULL; + + ret_code = create_mg_test_graph( + p_handle, h_src, h_dst, h_wgt, num_edges, store_transposed, FALSE, &p_graph, &ret_error); + + ret_code = cugraph_sssp(p_handle, p_graph, source, cutoff, TRUE, FALSE, &p_result, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, cugraph_error_message(ret_error)); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "cugraph_sssp failed."); + + cugraph_type_erased_device_array_view_t* vertices; + cugraph_type_erased_device_array_view_t* distances; + cugraph_type_erased_device_array_view_t* predecessors; + + vertices = cugraph_paths_result_get_vertices(p_result); + distances = cugraph_paths_result_get_distances(p_result); + predecessors = cugraph_paths_result_get_predecessors(p_result); + + vertex_t h_vertices[num_vertices]; + weight_t h_distances[num_vertices]; + vertex_t h_predecessors[num_vertices]; + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + p_handle, (byte_t*)h_vertices, vertices, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + p_handle, (byte_t*)h_distances, distances, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + ret_code = cugraph_type_erased_device_array_view_copy_to_host( + p_handle, (byte_t*)h_predecessors, predecessors, &ret_error); + TEST_ASSERT(test_ret_value, ret_code == CUGRAPH_SUCCESS, "copy_to_host failed."); + + size_t num_local_vertices = cugraph_type_erased_device_array_view_size(vertices); + + for (int i = 0; (i < num_local_vertices) && (test_ret_value == 0); ++i) { + TEST_ASSERT(test_ret_value, + nearlyEqual(expected_distances[h_vertices[i]], h_distances[i], EPSILON), + "sssp distances don't match"); + + TEST_ASSERT(test_ret_value, + expected_predecessors[h_vertices[i]] == h_predecessors[i], + "sssp predecessors don't match"); + } + + cugraph_type_erased_device_array_view_free(vertices); + cugraph_type_erased_device_array_view_free(distances); + cugraph_type_erased_device_array_view_free(predecessors); + cugraph_paths_result_free(p_result); + cugraph_sg_graph_free(p_graph); + cugraph_error_free(ret_error); + + return test_ret_value; +} + +int test_sssp(const cugraph_resource_handle_t* p_handle) +{ + size_t num_edges = 8; + size_t num_vertices = 6; + + vertex_t src[] = {0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t dst[] = {1, 3, 4, 0, 1, 3, 5, 5}; + weight_t wgt[] = {0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f}; + weight_t expected_distances[] = {0.0f, 0.1f, FLT_MAX, 2.2f, 1.2f, 4.4f}; + vertex_t expected_predecessors[] = {-1, 0, -1, 1, 1, 4}; + + // Bfs wants store_transposed = FALSE + return generic_sssp_test(p_handle, + src, + dst, + wgt, + 0, + expected_distances, + expected_predecessors, + num_vertices, + num_edges, + 10, + FALSE); +} + +int test_sssp_with_transpose(const cugraph_resource_handle_t* p_handle) +{ + size_t num_edges = 8; + size_t num_vertices = 6; + + vertex_t src[] = {0, 1, 1, 2, 2, 2, 3, 4}; + vertex_t dst[] = {1, 3, 4, 0, 1, 3, 5, 5}; + weight_t wgt[] = {0.1f, 2.1f, 1.1f, 5.1f, 3.1f, 4.1f, 7.2f, 3.2f}; + weight_t expected_distances[] = {0.0f, 0.1f, FLT_MAX, 2.2f, 1.2f, 4.4f}; + vertex_t expected_predecessors[] = {-1, 0, -1, 1, 1, 4}; + + // Bfs wants store_transposed = FALSE + // This call will force cugraph_sssp to transpose the graph + return generic_sssp_test(p_handle, + src, + dst, + wgt, + 0, + expected_distances, + expected_predecessors, + num_vertices, + num_edges, + 10, + TRUE); +} + +/******************************************************************************/ + +int main(int argc, char** argv) +{ + // Set up MPI: + int comm_rank; + int comm_size; + int num_gpus_per_node; + cudaError_t status; + int mpi_status; + cugraph_resource_handle_t* handle = NULL; + cugraph_error_t* ret_error; + cugraph_error_code_t ret_code = CUGRAPH_SUCCESS; + int prows = 1; + + C_MPI_TRY(MPI_Init(&argc, &argv)); + C_MPI_TRY(MPI_Comm_rank(MPI_COMM_WORLD, &comm_rank)); + C_MPI_TRY(MPI_Comm_size(MPI_COMM_WORLD, &comm_size)); + C_CUDA_TRY(cudaGetDeviceCount(&num_gpus_per_node)); + C_CUDA_TRY(cudaSetDevice(comm_rank % num_gpus_per_node)); + + void* raft_handle = create_raft_handle(prows); + handle = cugraph_create_resource_handle(raft_handle); + int result = 0; + result |= RUN_MG_TEST(test_sssp, handle); + result |= RUN_MG_TEST(test_sssp_with_transpose, handle); + + cugraph_free_resource_handle(handle); + free_raft_handle(raft_handle); + + C_MPI_TRY(MPI_Finalize()); + + return result; +} From 969a56ab7a0fe996c4be39e0be6dc2cb47fb9adb Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 20 May 2022 14:05:47 +0000 Subject: [PATCH 07/14] various fixes --- .gitignore | 1 + cpp/tests/CMakeLists.txt | 2 +- python/.gitignore | 1 - python/cufile.log | 206 ------ .../cugraph/traversal/sssp_wrapper.pyx | 2 +- python/pylibcugraph/pylibcugraph/__init__.py | 2 +- testing/cufile.log | 608 ++++++++++++++++++ testing/dask-worker-space/global.lock | 0 testing/dask-worker-space/purge.lock | 0 testing/testing2.ipynb | 561 ++++++++++++++++ 10 files changed, 1173 insertions(+), 210 deletions(-) delete mode 100644 python/cufile.log create mode 100644 testing/cufile.log create mode 100644 testing/dask-worker-space/global.lock create mode 100644 testing/dask-worker-space/purge.lock create mode 100644 testing/testing2.ipynb diff --git a/.gitignore b/.gitignore index 485b71cac68..215dfb2922a 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,4 @@ python/cugraph/cugraph/tests/dask-worker-space # Sphinx docs & build artifacts docs/cugraph/source/api_docs/api/* + diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index b5900a73291..3c71dfffd5c 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -1,4 +1,4 @@ -z#============================================================================= +#============================================================================= # Copyright (c) 2019-2022, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/python/.gitignore b/python/.gitignore index 49c8f3c3aeb..23a3af14582 100644 --- a/python/.gitignore +++ b/python/.gitignore @@ -1,3 +1,2 @@ *.cpp __pycache__ -testing diff --git a/python/cufile.log b/python/cufile.log deleted file mode 100644 index c65191d44cd..00000000000 --- a/python/cufile.log +++ /dev/null @@ -1,206 +0,0 @@ - 17-05-2022 18:03:11:622 [pid=3338 tid=3338] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:03:11:991 [pid=3500 tid=3920] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:03:11:994 [pid=3497 tid=3924] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:03:11:994 [pid=3503 tid=3922] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:03:11:995 [pid=3480 tid=3921] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:03:11:995 [pid=3490 tid=3926] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:03:11:995 [pid=3484 tid=3925] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:03:11:996 [pid=3489 tid=3927] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:03:11:997 [pid=3494 tid=3923] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:41:34:494 [pid=43612 tid=43612] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:41:34:670 [pid=43772 tid=44183] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:41:34:670 [pid=43762 tid=44185] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:41:34:670 [pid=43754 tid=44184] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:41:34:671 [pid=43769 tid=44188] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:41:34:671 [pid=43766 tid=44189] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:41:34:671 [pid=43761 tid=44187] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:41:34:671 [pid=43757 tid=44186] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:41:34:671 [pid=43749 tid=44190] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:45:05:225 [pid=46458 tid=46905] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:45:05:227 [pid=46456 tid=46906] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:45:05:228 [pid=46471 tid=46907] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:45:05:229 [pid=46451 tid=46910] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:45:05:229 [pid=46462 tid=46908] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:45:05:229 [pid=46465 tid=46911] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:45:05:229 [pid=46468 tid=46912] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 18:45:05:229 [pid=46474 tid=46909] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:13:15:122 [pid=69473 tid=69473] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:13:15:274 [pid=69646 tid=70053] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:13:15:274 [pid=69623 tid=70054] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:13:15:275 [pid=69640 tid=70056] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:13:15:275 [pid=69643 tid=70055] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:13:15:275 [pid=69631 tid=70059] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:13:15:276 [pid=69637 tid=70058] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:13:15:276 [pid=69627 tid=70057] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:13:15:277 [pid=69634 tid=70060] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:21:36:350 [pid=74778 tid=74778] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:21:36:555 [pid=74952 tid=75382] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:21:36:559 [pid=74940 tid=75383] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:21:36:559 [pid=74943 tid=75385] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:21:36:560 [pid=74946 tid=75386] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:21:36:560 [pid=74929 tid=75384] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:21:36:561 [pid=74933 tid=75388] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:21:36:561 [pid=74937 tid=75387] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:21:36:562 [pid=74949 tid=75389] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:22:20:322 [pid=75622 tid=75622] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:22:20:475 [pid=75779 tid=76207] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:22:20:475 [pid=75795 tid=76208] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:22:20:476 [pid=75781 tid=76210] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:22:20:476 [pid=75786 tid=76209] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:22:20:477 [pid=75797 tid=76212] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:22:20:477 [pid=75791 tid=76211] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:22:20:478 [pid=75788 tid=76213] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:22:20:479 [pid=75774 tid=76214] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:37:03:382 [pid=6382 tid=6382] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:37:03:527 [pid=6531 tid=6969] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:37:03:527 [pid=6547 tid=6970] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:37:03:528 [pid=6539 tid=6973] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:37:03:528 [pid=6527 tid=6971] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:37:03:529 [pid=6544 tid=6975] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:37:03:530 [pid=6534 tid=6974] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:37:03:530 [pid=6550 tid=6972] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:37:03:530 [pid=6540 tid=6976] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:38:12:678 [pid=7440 tid=7440] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:38:12:854 [pid=7608 tid=8025] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:38:12:856 [pid=7605 tid=8028] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:38:12:857 [pid=7591 tid=8026] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:38:12:857 [pid=7611 tid=8027] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:38:12:857 [pid=7602 tid=8029] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:38:12:857 [pid=7599 tid=8032] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:38:12:858 [pid=7588 tid=8030] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:38:12:858 [pid=7596 tid=8031] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:54:27:169 [pid=15967 tid=15967] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:54:27:362 [pid=16138 tid=16550] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:54:27:363 [pid=16122 tid=16551] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:54:27:363 [pid=16135 tid=16552] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:54:27:363 [pid=16129 tid=16554] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:54:27:363 [pid=16126 tid=16553] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:54:27:363 [pid=16132 tid=16555] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:54:27:364 [pid=16120 tid=16557] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 19:54:27:365 [pid=16115 tid=16556] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:08:51:697 [pid=24245 tid=24245] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:08:51:888 [pid=24420 tid=24844] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:08:51:888 [pid=24408 tid=24845] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:08:51:888 [pid=24404 tid=24846] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:08:51:889 [pid=24402 tid=24849] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:08:51:889 [pid=24414 tid=24847] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:08:51:889 [pid=24397 tid=24848] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:08:51:890 [pid=24411 tid=24850] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:08:51:891 [pid=24417 tid=24851] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:21:26:220 [pid=31544 tid=31544] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:21:26:434 [pid=31693 tid=32115] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:21:26:436 [pid=31704 tid=32116] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:21:26:436 [pid=31695 tid=32117] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:21:26:436 [pid=31698 tid=32118] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:21:26:437 [pid=31685 tid=32119] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:21:26:437 [pid=31681 tid=32120] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:21:26:437 [pid=31688 tid=32121] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:21:26:438 [pid=31701 tid=32122] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:43:26:771 [pid=43701 tid=43701] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:43:26:967 [pid=43858 tid=44270] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:43:26:967 [pid=43850 tid=44269] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:43:26:969 [pid=43838 tid=44272] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:43:26:969 [pid=43852 tid=44271] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:43:26:969 [pid=43847 tid=44275] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:43:26:969 [pid=43855 tid=44273] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:43:26:970 [pid=43861 tid=44274] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:43:26:971 [pid=43842 tid=44276] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:55:08:49 [pid=50394 tid=50394] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:55:08:246 [pid=50543 tid=50970] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:55:08:250 [pid=50546 tid=50971] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:55:08:250 [pid=50552 tid=50973] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:55:08:250 [pid=50549 tid=50972] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:55:08:251 [pid=50529 tid=50975] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:55:08:251 [pid=50537 tid=50976] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:55:08:251 [pid=50532 tid=50977] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 20:55:08:251 [pid=50540 tid=50974] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:11:17:214 [pid=59558 tid=59558] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:11:17:395 [pid=59733 tid=60154] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:11:17:397 [pid=59727 tid=60155] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:11:17:397 [pid=59714 tid=60158] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:11:17:398 [pid=59723 tid=60156] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:11:17:398 [pid=59721 tid=60157] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:11:17:398 [pid=59730 tid=60159] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:11:17:398 [pid=59718 tid=60161] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:11:17:398 [pid=59710 tid=60160] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:14:56:768 [pid=62202 tid=62202] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:14:56:947 [pid=62368 tid=62787] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:14:56:950 [pid=62366 tid=62789] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:14:56:950 [pid=62357 tid=62788] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:14:56:950 [pid=62362 tid=62790] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:14:56:951 [pid=62374 tid=62791] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:14:56:951 [pid=62371 tid=62792] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:14:56:953 [pid=62377 tid=62794] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:14:56:953 [pid=62354 tid=62793] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:17:07:458 [pid=63920 tid=63920] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:17:07:643 [pid=64090 tid=64504] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:17:07:646 [pid=64084 tid=64506] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:17:07:646 [pid=64070 tid=64509] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:17:07:646 [pid=64081 tid=64507] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:17:07:646 [pid=64093 tid=64505] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:17:07:646 [pid=64075 tid=64508] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:17:07:647 [pid=64087 tid=64510] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:17:07:648 [pid=64077 tid=64511] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:19:32:30 [pid=65731 tid=65731] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:19:32:213 [pid=65888 tid=66330] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:19:32:213 [pid=65899 tid=66329] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:19:32:214 [pid=65911 tid=66331] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:19:32:214 [pid=65902 tid=66332] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:19:32:215 [pid=65895 tid=66333] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:19:32:215 [pid=65908 tid=66335] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:19:32:215 [pid=65905 tid=66334] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 17-05-2022 21:19:32:217 [pid=65893 tid=66336] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:34:21:262 [pid=54551 tid=54551] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:34:21:449 [pid=54792 tid=55307] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:34:21:450 [pid=54778 tid=55308] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:34:21:450 [pid=54789 tid=55311] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:34:21:450 [pid=54804 tid=55309] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:34:21:451 [pid=54782 tid=55313] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:34:21:451 [pid=54795 tid=55312] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:34:21:451 [pid=54799 tid=55314] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:34:21:451 [pid=54786 tid=55310] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:05:16:392 [pid=71498 tid=71498] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:05:16:582 [pid=71659 tid=72080] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:05:16:583 [pid=71653 tid=72081] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:05:16:585 [pid=71664 tid=72082] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:05:16:585 [pid=71674 tid=72083] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:05:16:586 [pid=71667 tid=72086] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:05:16:586 [pid=71661 tid=72085] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:05:16:586 [pid=71650 tid=72084] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:05:16:587 [pid=71677 tid=72087] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:17:42:619 [pid=79259 tid=79259] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:17:42:825 [pid=79425 tid=79855] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:17:42:826 [pid=79437 tid=79857] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:17:42:826 [pid=79434 tid=79856] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:17:42:827 [pid=79431 tid=79859] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:17:42:828 [pid=79414 tid=79858] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:17:42:828 [pid=79427 tid=79860] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:17:42:829 [pid=79418 tid=79861] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:17:42:829 [pid=79410 tid=79862] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:47:31:360 [pid=17762 tid=17762] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:47:31:545 [pid=17933 tid=18357] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:47:31:546 [pid=17922 tid=18358] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:47:31:548 [pid=17945 tid=18359] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:47:31:548 [pid=17939 tid=18360] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:47:31:549 [pid=17936 tid=18361] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:47:31:549 [pid=17942 tid=18362] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:47:31:550 [pid=17926 tid=18363] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:47:31:551 [pid=17930 tid=18364] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:55:45:548 [pid=23418 tid=23418] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:55:45:738 [pid=23596 tid=24009] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:55:45:739 [pid=23577 tid=24011] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:55:45:739 [pid=23579 tid=24010] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:55:45:739 [pid=23571 tid=24012] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:55:45:739 [pid=23582 tid=24014] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:55:45:739 [pid=23588 tid=24013] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:55:45:740 [pid=23585 tid=24015] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:55:45:741 [pid=23565 tid=24016] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:59:33:412 [pid=26276 tid=26276] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:59:33:603 [pid=26452 tid=26872] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:59:33:603 [pid=26437 tid=26870] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:59:33:603 [pid=26450 tid=26868] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:59:33:603 [pid=26440 tid=26869] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:59:33:603 [pid=26447 tid=26871] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:59:33:603 [pid=26444 tid=26873] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:59:33:604 [pid=26456 tid=26874] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:59:33:604 [pid=26433 tid=26875] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded diff --git a/python/cugraph/cugraph/traversal/sssp_wrapper.pyx b/python/cugraph/cugraph/traversal/sssp_wrapper.pyx index 6972cd40b68..b83f0ab7a7b 100644 --- a/python/cugraph/cugraph/traversal/sssp_wrapper.pyx +++ b/python/cugraph/cugraph/traversal/sssp_wrapper.pyx @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2021, NVIDIA CORPORATION. +# Copyright (c) 2019-2022, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/python/pylibcugraph/pylibcugraph/__init__.py b/python/pylibcugraph/pylibcugraph/__init__.py index 2406cfbdf5b..0a5bdc7cb66 100644 --- a/python/pylibcugraph/pylibcugraph/__init__.py +++ b/python/pylibcugraph/pylibcugraph/__init__.py @@ -35,4 +35,4 @@ from pylibcugraph.node2vec import node2vec -from pylibcugraph.bfs import bfs \ No newline at end of file +from pylibcugraph.bfs import bfs diff --git a/testing/cufile.log b/testing/cufile.log new file mode 100644 index 00000000000..17da161a231 --- /dev/null +++ b/testing/cufile.log @@ -0,0 +1,608 @@ + 18-05-2022 20:33:46:285 [pid=29501 tid=29847] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:33:46:291 [pid=29508 tid=29848] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:33:46:291 [pid=29487 tid=29849] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:33:46:291 [pid=29504 tid=29852] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:33:46:291 [pid=29499 tid=29850] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:33:46:291 [pid=29492 tid=29853] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:33:46:292 [pid=29510 tid=29851] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:33:46:293 [pid=29496 tid=29854] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:40:53:260 [pid=33667 tid=34102] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:40:53:261 [pid=33673 tid=34103] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:40:53:262 [pid=33677 tid=34104] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:40:53:262 [pid=33655 tid=34108] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:40:53:263 [pid=33660 tid=34105] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:40:53:263 [pid=33668 tid=34106] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:40:53:263 [pid=33664 tid=34107] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:40:53:264 [pid=33672 tid=34109] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:56:51:316 [pid=43329 tid=43755] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:56:51:317 [pid=43320 tid=43757] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:56:51:317 [pid=43325 tid=43756] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:56:51:319 [pid=43313 tid=43760] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:56:51:319 [pid=43312 tid=43759] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:56:51:319 [pid=43303 tid=43758] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:56:51:320 [pid=43308 tid=43761] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 20:56:51:320 [pid=43318 tid=43762] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:03:31:46 [pid=48019 tid=48460] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:03:31:52 [pid=48028 tid=48463] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:03:31:52 [pid=48038 tid=48461] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:03:31:52 [pid=48041 tid=48465] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:03:31:52 [pid=48033 tid=48464] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:03:31:52 [pid=48034 tid=48466] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:03:31:52 [pid=48023 tid=48462] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:03:31:53 [pid=48030 tid=48467] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:09:28:313 [pid=52083 tid=52518] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:09:28:314 [pid=52094 tid=52520] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:09:28:315 [pid=52100 tid=52521] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:09:28:315 [pid=52091 tid=52524] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:09:28:315 [pid=52080 tid=52519] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:09:28:315 [pid=52097 tid=52525] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:09:28:315 [pid=52088 tid=52522] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:09:28:315 [pid=52105 tid=52523] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:11:15:590 [pid=53538 tid=53962] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:11:15:591 [pid=53531 tid=53963] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:11:15:592 [pid=53530 tid=53966] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:11:15:592 [pid=53543 tid=53969] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:11:15:592 [pid=53521 tid=53965] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:11:15:592 [pid=53526 tid=53964] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:11:15:592 [pid=53532 tid=53967] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:11:15:593 [pid=53540 tid=53968] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:14:27:272 [pid=55658 tid=56095] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:14:27:273 [pid=55663 tid=56096] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:14:27:273 [pid=55668 tid=56097] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:14:27:273 [pid=55646 tid=56099] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:14:27:273 [pid=55649 tid=56098] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:14:27:275 [pid=55665 tid=56102] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:14:27:275 [pid=55657 tid=56101] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:14:27:275 [pid=55653 tid=56100] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:22:09:246 [pid=60251 tid=60694] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:22:09:247 [pid=60265 tid=60697] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:22:09:248 [pid=60263 tid=60698] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:22:09:248 [pid=60260 tid=60699] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:22:09:248 [pid=60254 tid=60696] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:22:09:248 [pid=60243 tid=60700] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:22:09:249 [pid=60256 tid=60695] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:22:09:249 [pid=60246 tid=60701] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:25:07:609 [pid=62455 tid=62899] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:25:07:611 [pid=62471 tid=62901] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:25:07:611 [pid=62464 tid=62900] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:25:07:611 [pid=62462 tid=62903] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:25:07:611 [pid=62459 tid=62902] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:25:07:612 [pid=62447 tid=62905] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:25:07:612 [pid=62453 tid=62904] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:25:07:613 [pid=62466 tid=62906] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:27:00:331 [pid=63933 tid=64348] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:27:00:331 [pid=63910 tid=64351] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:27:00:332 [pid=63926 tid=64349] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:27:00:332 [pid=63921 tid=64350] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:27:00:332 [pid=63913 tid=64352] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:27:00:333 [pid=63929 tid=64353] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:27:00:333 [pid=63923 tid=64355] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:27:00:333 [pid=63918 tid=64354] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:29:06:532 [pid=65525 tid=65954] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:29:06:533 [pid=65520 tid=65955] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:29:06:533 [pid=65517 tid=65956] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:29:06:533 [pid=65507 tid=65959] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:29:06:534 [pid=65511 tid=65957] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:29:06:534 [pid=65503 tid=65958] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:29:06:534 [pid=65513 tid=65960] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:29:06:535 [pid=65522 tid=65961] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:32:02:555 [pid=67503 tid=67940] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:32:02:556 [pid=67522 tid=67941] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:32:02:556 [pid=67507 tid=67942] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:32:02:558 [pid=67529 tid=67944] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:32:02:558 [pid=67516 tid=67943] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:32:02:558 [pid=67512 tid=67946] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:32:02:558 [pid=67520 tid=67945] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:32:02:559 [pid=67515 tid=67947] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:33:53:741 [pid=69000 tid=69450] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:33:53:742 [pid=69021 tid=69451] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:33:53:743 [pid=69010 tid=69452] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:33:53:743 [pid=69006 tid=69453] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:33:53:743 [pid=69019 tid=69454] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:33:53:744 [pid=69015 tid=69456] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:33:53:745 [pid=69012 tid=69455] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:33:53:745 [pid=69024 tid=69457] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:36:02:706 [pid=70640 tid=71056] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:36:02:708 [pid=70623 tid=71058] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:36:02:708 [pid=70618 tid=71057] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:36:02:709 [pid=70625 tid=71060] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:36:02:709 [pid=70635 tid=71059] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:36:02:709 [pid=70637 tid=71061] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:36:02:710 [pid=70632 tid=71062] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:36:02:710 [pid=70629 tid=71063] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:39:51:49 [pid=73079 tid=73499] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:39:51:49 [pid=73078 tid=73500] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:39:51:49 [pid=73057 tid=73502] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:39:51:50 [pid=73067 tid=73501] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:39:51:50 [pid=73075 tid=73503] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:39:51:51 [pid=73063 tid=73505] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:39:51:51 [pid=73073 tid=73504] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:39:51:51 [pid=73054 tid=73506] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:51:28:124 [pid=79799 tid=80239] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:51:28:126 [pid=79802 tid=80240] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:51:28:126 [pid=79791 tid=80241] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:51:28:127 [pid=79814 tid=80242] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:51:28:127 [pid=79804 tid=80243] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:51:28:127 [pid=79794 tid=80245] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:51:28:127 [pid=79811 tid=80244] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:51:28:128 [pid=79807 tid=80246] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:52:27:910 [pid=80826 tid=81257] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:52:27:910 [pid=80817 tid=81258] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:52:27:911 [pid=80824 tid=81260] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:52:27:912 [pid=80815 tid=81261] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:52:27:912 [pid=80809 tid=81259] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:52:27:912 [pid=80804 tid=81262] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:52:27:914 [pid=80819 tid=81264] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:52:27:914 [pid=80814 tid=81263] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:57:17:775 [pid=2164 tid=2606] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:57:17:775 [pid=2167 tid=2607] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:57:17:776 [pid=2153 tid=2609] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:57:17:777 [pid=2157 tid=2610] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:57:17:777 [pid=2172 tid=2611] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:57:17:778 [pid=2169 tid=2608] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:57:17:779 [pid=2159 tid=2612] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 21:57:17:780 [pid=2175 tid=2613] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:02:07:460 [pid=5167 tid=5606] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:02:07:463 [pid=5163 tid=5607] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:02:07:463 [pid=5173 tid=5610] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:02:07:464 [pid=5154 tid=5609] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:02:07:464 [pid=5162 tid=5612] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:02:07:464 [pid=5176 tid=5611] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:02:07:464 [pid=5171 tid=5608] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:02:07:464 [pid=5157 tid=5613] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:05:53:105 [pid=7772 tid=8211] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:05:53:108 [pid=7779 tid=8213] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:05:53:109 [pid=7785 tid=8212] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:05:53:109 [pid=7770 tid=8216] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:05:53:109 [pid=7767 tid=8214] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:05:53:109 [pid=7777 tid=8217] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:05:53:110 [pid=7763 tid=8215] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:05:53:112 [pid=7783 tid=8218] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:07:08:763 [pid=8938 tid=9387] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:07:08:763 [pid=8956 tid=9386] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:07:08:763 [pid=8941 tid=9391] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:07:08:763 [pid=8948 tid=9385] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:07:08:763 [pid=8952 tid=9390] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:07:08:764 [pid=8959 tid=9389] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:07:08:765 [pid=8951 tid=9388] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:07:08:766 [pid=8943 tid=9392] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:12:05:679 [pid=12245 tid=12684] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:12:05:680 [pid=12258 tid=12685] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:12:05:680 [pid=12249 tid=12687] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:12:05:681 [pid=12251 tid=12689] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:12:05:682 [pid=12267 tid=12690] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:12:05:682 [pid=12256 tid=12693] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:12:05:682 [pid=12261 tid=12691] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:12:05:682 [pid=12264 tid=12692] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:30:20:23 [pid=22054 tid=22470] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:30:20:24 [pid=22048 tid=22471] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:30:20:25 [pid=22033 tid=22472] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:30:20:25 [pid=22036 tid=22474] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:30:20:26 [pid=22042 tid=22473] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:30:20:26 [pid=22045 tid=22475] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:30:20:27 [pid=22029 tid=22476] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:30:20:27 [pid=22051 tid=22477] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:32:39:898 [pid=23591 tid=24054] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:32:39:900 [pid=23586 tid=24055] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:32:39:901 [pid=23578 tid=24056] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:32:39:901 [pid=23600 tid=24057] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:32:39:902 [pid=23587 tid=24059] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:32:39:902 [pid=23597 tid=24058] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:32:39:903 [pid=23594 tid=24060] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:32:39:907 [pid=23580 tid=24061] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:37:56:507 [pid=27108 tid=27544] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:37:56:508 [pid=27102 tid=27546] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:37:56:509 [pid=27095 tid=27548] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:37:56:509 [pid=27112 tid=27545] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:37:56:509 [pid=27115 tid=27549] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:37:56:509 [pid=27098 tid=27547] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:37:56:510 [pid=27117 tid=27550] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:37:56:510 [pid=27106 tid=27551] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:40:55:977 [pid=29238 tid=29675] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:40:55:977 [pid=29244 tid=29674] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:40:55:979 [pid=29229 tid=29676] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:40:55:979 [pid=29235 tid=29678] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:40:55:979 [pid=29223 tid=29677] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:40:55:980 [pid=29225 tid=29680] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:40:55:980 [pid=29242 tid=29681] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:40:55:980 [pid=29233 tid=29679] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:43:18:116 [pid=31082 tid=31523] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:43:18:118 [pid=31085 tid=31524] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:43:18:120 [pid=31089 tid=31525] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:43:18:120 [pid=31101 tid=31526] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:43:18:120 [pid=31078 tid=31528] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:43:18:121 [pid=31104 tid=31527] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:43:18:122 [pid=31092 tid=31529] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:43:18:122 [pid=31094 tid=31530] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:45:39:700 [pid=32813 tid=33240] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:45:39:701 [pid=32806 tid=33243] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:45:39:701 [pid=32824 tid=33241] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:45:39:701 [pid=32821 tid=33242] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:45:39:701 [pid=32817 tid=33244] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:45:39:701 [pid=32801 tid=33245] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:45:39:701 [pid=32811 tid=33246] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:45:39:701 [pid=32818 tid=33247] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:47:30:634 [pid=34277 tid=34707] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:47:30:636 [pid=34273 tid=34709] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:47:30:636 [pid=34263 tid=34708] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:47:30:636 [pid=34278 tid=34711] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:47:30:636 [pid=34281 tid=34710] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:47:30:636 [pid=34270 tid=34712] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:47:30:637 [pid=34267 tid=34713] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:47:30:637 [pid=34259 tid=34714] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:49:06:710 [pid=35613 tid=36047] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:49:06:710 [pid=35619 tid=36048] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:49:06:710 [pid=35630 tid=36049] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:49:06:710 [pid=35608 tid=36051] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:49:06:711 [pid=35621 tid=36050] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:49:06:711 [pid=35627 tid=36052] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:49:06:712 [pid=35617 tid=36053] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:49:06:713 [pid=35624 tid=36054] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:50:24:407 [pid=36788 tid=37214] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:50:24:408 [pid=36794 tid=37213] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:50:24:409 [pid=36780 tid=37215] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:50:24:409 [pid=36782 tid=37216] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:50:24:410 [pid=36771 tid=37218] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:50:24:410 [pid=36785 tid=37219] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:50:24:410 [pid=36775 tid=37217] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:50:24:410 [pid=36791 tid=37220] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:57:20:181 [pid=40857 tid=41302] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:57:20:181 [pid=40877 tid=41303] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:57:20:181 [pid=40854 tid=41301] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:57:20:181 [pid=40864 tid=41300] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:57:20:181 [pid=40873 tid=41306] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:57:20:181 [pid=40870 tid=41299] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:57:20:182 [pid=40867 tid=41305] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 22:57:20:182 [pid=40862 tid=41304] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:03:17:277 [pid=44777 tid=45205] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:03:17:280 [pid=44771 tid=45206] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:03:17:281 [pid=44764 tid=45208] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:03:17:281 [pid=44780 tid=45207] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:03:17:281 [pid=44783 tid=45210] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:03:17:282 [pid=44786 tid=45209] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:03:17:282 [pid=44768 tid=45211] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:03:17:283 [pid=44775 tid=45212] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:07:15:271 [pid=47725 tid=48156] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:07:15:274 [pid=47734 tid=48157] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:07:15:274 [pid=47712 tid=48159] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:07:15:275 [pid=47719 tid=48161] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:07:15:275 [pid=47716 tid=48160] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:07:15:275 [pid=47729 tid=48158] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:07:15:275 [pid=47731 tid=48162] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:07:15:276 [pid=47723 tid=48163] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:11:22:946 [pid=50710 tid=51128] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:11:22:946 [pid=50706 tid=51127] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:11:22:946 [pid=50700 tid=51129] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:11:22:948 [pid=50689 tid=51130] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:11:22:949 [pid=50697 tid=51131] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:11:22:949 [pid=50709 tid=51132] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:11:22:950 [pid=50692 tid=51133] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:11:22:950 [pid=50703 tid=51134] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:14:06:999 [pid=52412 tid=52877] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:14:07:0 [pid=52418 tid=52880] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:14:07:1 [pid=52428 tid=52878] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:14:07:1 [pid=52417 tid=52879] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:14:07:1 [pid=52425 tid=52881] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:14:07:2 [pid=52409 tid=52883] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:14:07:3 [pid=52423 tid=52884] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:14:07:3 [pid=52431 tid=52882] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:17:55:635 [pid=54901 tid=55346] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:17:55:635 [pid=54895 tid=55347] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:17:55:637 [pid=54903 tid=55349] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:17:55:638 [pid=54906 tid=55348] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:17:55:638 [pid=54884 tid=55350] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:17:55:639 [pid=54887 tid=55351] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:17:55:639 [pid=54898 tid=55352] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:17:55:639 [pid=54893 tid=55353] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:26:04:858 [pid=59631 tid=60094] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:26:04:858 [pid=59626 tid=60092] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:26:04:858 [pid=59624 tid=60091] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:26:04:858 [pid=59639 tid=60093] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:26:04:858 [pid=59622 tid=60095] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:26:04:859 [pid=59637 tid=60096] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:26:04:860 [pid=59634 tid=60097] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:26:04:860 [pid=59616 tid=60098] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:27:22:787 [pid=60784 tid=61259] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:27:22:793 [pid=60780 tid=61260] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:27:22:793 [pid=60792 tid=61261] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:27:22:794 [pid=60800 tid=61262] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:27:22:795 [pid=60789 tid=61265] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:27:22:795 [pid=60797 tid=61263] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:27:22:795 [pid=60793 tid=61264] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:27:22:796 [pid=60803 tid=61266] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:29:11:578 [pid=62080 tid=62542] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:29:11:580 [pid=62083 tid=62543] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:29:11:580 [pid=62067 tid=62544] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:29:11:581 [pid=62075 tid=62545] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:29:11:582 [pid=62071 tid=62546] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:29:11:582 [pid=62089 tid=62547] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:29:11:582 [pid=62077 tid=62548] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:29:11:583 [pid=62086 tid=62549] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:30:23:221 [pid=63201 tid=63657] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:30:23:222 [pid=63183 tid=63658] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:30:23:222 [pid=63198 tid=63659] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:30:23:223 [pid=63190 tid=63661] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:30:23:223 [pid=63193 tid=63660] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:30:23:224 [pid=63179 tid=63662] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:30:23:224 [pid=63195 tid=63664] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:30:23:224 [pid=63187 tid=63663] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:32:30:548 [pid=65149 tid=65618] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:32:30:549 [pid=65143 tid=65619] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:32:30:550 [pid=65139 tid=65621] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:32:30:550 [pid=65157 tid=65620] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:32:30:550 [pid=65148 tid=65622] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:32:30:551 [pid=65164 tid=65623] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:32:30:551 [pid=65167 tid=65625] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:32:30:552 [pid=65152 tid=65624] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:38:23:257 [pid=68756 tid=69212] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:38:23:260 [pid=68754 tid=69213] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:38:23:260 [pid=68734 tid=69214] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:38:23:261 [pid=68751 tid=69215] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:38:23:261 [pid=68747 tid=69217] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:38:23:262 [pid=68742 tid=69218] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:38:23:262 [pid=68744 tid=69216] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:38:23:262 [pid=68738 tid=69219] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:40:15:510 [pid=70220 tid=70674] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:40:15:511 [pid=70214 tid=70675] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:40:15:512 [pid=70218 tid=70676] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:40:15:513 [pid=70208 tid=70679] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:40:15:513 [pid=70211 tid=70677] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:40:15:513 [pid=70195 tid=70678] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:40:15:514 [pid=70199 tid=70680] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:40:15:514 [pid=70206 tid=70681] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:44:38:239 [pid=72965 tid=73418] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:44:38:241 [pid=72957 tid=73419] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:44:38:242 [pid=72959 tid=73423] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:44:38:242 [pid=72968 tid=73424] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:44:38:242 [pid=72943 tid=73420] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:44:38:242 [pid=72962 tid=73421] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:44:38:242 [pid=72951 tid=73422] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:44:38:243 [pid=72947 tid=73425] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:46:03:522 [pid=74182 tid=74647] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:46:03:523 [pid=74175 tid=74648] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:46:03:525 [pid=74180 tid=74649] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:46:03:525 [pid=74192 tid=74650] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:46:03:525 [pid=74197 tid=74652] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:46:03:526 [pid=74195 tid=74651] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:46:03:526 [pid=74190 tid=74653] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:46:03:526 [pid=74187 tid=74654] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:57:00:611 [pid=80513 tid=80975] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:57:00:611 [pid=80523 tid=80976] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:57:00:612 [pid=80528 tid=80977] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:57:00:612 [pid=80518 tid=80978] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:57:00:613 [pid=80505 tid=80980] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:57:00:613 [pid=80510 tid=80979] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:57:00:614 [pid=80525 tid=80981] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:57:00:614 [pid=80519 tid=80982] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:58:57:597 [pid=544 tid=1005] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:58:57:597 [pid=533 tid=1006] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:58:57:599 [pid=555 tid=1007] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:58:57:599 [pid=553 tid=1010] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:58:57:599 [pid=547 tid=1008] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:58:57:600 [pid=558 tid=1009] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:58:57:600 [pid=542 tid=1011] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 18-05-2022 23:58:57:601 [pid=538 tid=1012] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:00:39:54 [pid=1930 tid=2392] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:00:39:54 [pid=1933 tid=2393] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:00:39:55 [pid=1919 tid=2394] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:00:39:55 [pid=1911 tid=2395] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:00:39:56 [pid=1914 tid=2396] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:00:39:56 [pid=1927 tid=2397] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:00:39:56 [pid=1925 tid=2399] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:00:39:57 [pid=1922 tid=2398] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:14:40:782 [pid=9971 tid=10450] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:14:40:784 [pid=9980 tid=10451] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:14:40:784 [pid=9989 tid=10452] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:14:40:784 [pid=9987 tid=10453] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:14:40:785 [pid=9964 tid=10454] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:14:40:785 [pid=9977 tid=10455] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:14:40:785 [pid=9975 tid=10456] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:14:40:786 [pid=9983 tid=10457] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:22:11:27 [pid=14342 tid=14796] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:22:11:28 [pid=14351 tid=14797] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:22:11:30 [pid=14337 tid=14798] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:22:11:31 [pid=14338 tid=14799] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:22:11:31 [pid=14333 tid=14800] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:22:11:32 [pid=14348 tid=14801] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:22:11:32 [pid=14329 tid=14803] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:22:11:32 [pid=14346 tid=14802] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:34:44:989 [pid=21462 tid=21941] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:34:44:991 [pid=21468 tid=21942] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:34:44:991 [pid=21471 tid=21943] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:34:44:991 [pid=21482 tid=21944] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:34:44:992 [pid=21460 tid=21945] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:34:44:992 [pid=21474 tid=21947] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:34:44:992 [pid=21476 tid=21946] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:34:44:993 [pid=21479 tid=21948] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:38:18:686 [pid=23812 tid=24323] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:38:18:690 [pid=23824 tid=24328] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:38:18:690 [pid=23819 tid=24327] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:38:18:690 [pid=23804 tid=24324] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:38:18:690 [pid=23827 tid=24325] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:38:18:691 [pid=23815 tid=24329] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:38:18:691 [pid=23820 tid=24326] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:38:18:691 [pid=23830 tid=24330] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:41:36:123 [pid=26025 tid=26497] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:41:36:124 [pid=26047 tid=26498] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:41:36:124 [pid=26030 tid=26499] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:41:36:124 [pid=26033 tid=26501] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:41:36:124 [pid=26041 tid=26500] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:41:36:125 [pid=26037 tid=26504] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:41:36:125 [pid=26050 tid=26502] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:41:36:125 [pid=26045 tid=26503] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:44:51:492 [pid=28419 tid=28898] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:44:51:492 [pid=28433 tid=28899] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:44:51:492 [pid=28428 tid=28897] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:44:51:492 [pid=28441 tid=28900] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:44:51:492 [pid=28423 tid=28901] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:44:51:493 [pid=28438 tid=28902] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:44:51:493 [pid=28435 tid=28903] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:44:51:494 [pid=28431 tid=28904] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:46:10:782 [pid=29633 tid=30084] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:46:10:782 [pid=29641 tid=30085] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:46:10:782 [pid=29634 tid=30086] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:46:10:783 [pid=29625 tid=30087] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:46:10:783 [pid=29627 tid=30088] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:46:10:783 [pid=29621 tid=30089] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:46:10:784 [pid=29624 tid=30090] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:46:10:785 [pid=29615 tid=30091] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:49:04:630 [pid=31759 tid=32218] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:49:04:630 [pid=31738 tid=32222] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:49:04:631 [pid=31761 tid=32221] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:49:04:631 [pid=31756 tid=32220] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:49:04:631 [pid=31743 tid=32224] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:49:04:631 [pid=31747 tid=32219] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:49:04:631 [pid=31751 tid=32223] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:49:04:631 [pid=31749 tid=32225] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:53:43:730 [pid=34660 tid=35111] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:53:43:731 [pid=34646 tid=35112] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:53:43:732 [pid=34665 tid=35113] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:53:43:733 [pid=34668 tid=35114] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:53:43:733 [pid=34656 tid=35117] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:53:43:733 [pid=34650 tid=35118] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:53:43:733 [pid=34662 tid=35116] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:53:43:734 [pid=34653 tid=35115] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:54:39:910 [pid=35642 tid=36095] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:54:39:912 [pid=35627 tid=36096] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:54:39:914 [pid=35622 tid=36097] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:54:39:914 [pid=35639 tid=36099] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:54:39:914 [pid=35634 tid=36098] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:54:39:915 [pid=35637 tid=36101] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:54:39:915 [pid=35646 tid=36100] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 00:54:39:915 [pid=35648 tid=36102] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:03:53:568 [pid=40924 tid=41389] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:03:53:570 [pid=40930 tid=41391] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:03:53:570 [pid=40935 tid=41392] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:03:53:570 [pid=40913 tid=41390] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:03:53:571 [pid=40933 tid=41393] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:03:53:572 [pid=40917 tid=41395] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:03:53:572 [pid=40925 tid=41394] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:03:53:573 [pid=40921 tid=41396] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:05:11:767 [pid=42105 tid=42575] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:05:11:767 [pid=42116 tid=42577] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:05:11:767 [pid=42097 tid=42578] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:05:11:768 [pid=42113 tid=42576] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:05:11:768 [pid=42111 tid=42579] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:05:11:768 [pid=42108 tid=42580] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:05:11:769 [pid=42119 tid=42581] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:05:11:769 [pid=42102 tid=42582] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:18:43:662 [pid=49974 tid=50417] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:18:43:662 [pid=49967 tid=50418] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:18:43:663 [pid=49947 tid=50419] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:18:43:663 [pid=49970 tid=50421] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:18:43:664 [pid=49960 tid=50420] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:18:43:664 [pid=49951 tid=50424] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:18:43:664 [pid=49961 tid=50422] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:18:43:664 [pid=49965 tid=50423] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:29:06:700 [pid=56026 tid=56482] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:29:06:701 [pid=56000 tid=56484] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:29:06:702 [pid=56007 tid=56483] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:29:06:702 [pid=56017 tid=56486] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:29:06:702 [pid=56004 tid=56485] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:29:06:704 [pid=56010 tid=56487] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:29:06:704 [pid=56024 tid=56488] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:29:06:704 [pid=56021 tid=56489] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:31:40:61 [pid=57827 tid=58300] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:31:40:64 [pid=57853 tid=58302] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:31:40:64 [pid=57839 tid=58301] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:31:40:64 [pid=57851 tid=58305] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:31:40:64 [pid=57842 tid=58303] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:31:40:65 [pid=57848 tid=58304] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:31:40:65 [pid=57845 tid=58307] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 19-05-2022 01:31:40:65 [pid=57834 tid=58306] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 02:25:58:86 [pid=49269 tid=49709] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 02:25:58:86 [pid=49289 tid=49710] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 02:25:58:87 [pid=49292 tid=49712] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 02:25:58:88 [pid=49280 tid=49711] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 02:25:58:88 [pid=49273 tid=49714] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 02:25:58:88 [pid=49286 tid=49713] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 02:25:58:88 [pid=49277 tid=49716] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 02:25:58:88 [pid=49283 tid=49715] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 02:30:21:153 [pid=52226 tid=52650] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 02:30:21:154 [pid=52232 tid=52651] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 02:30:21:155 [pid=52221 tid=52653] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 02:30:21:156 [pid=52214 tid=52657] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 02:30:21:156 [pid=52210 tid=52654] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 02:30:21:156 [pid=52223 tid=52652] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 02:30:21:156 [pid=52229 tid=52655] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 02:30:21:156 [pid=52218 tid=52656] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:11:27:205 [pid=75511 tid=75952] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:11:27:208 [pid=75522 tid=75954] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:11:27:208 [pid=75515 tid=75953] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:11:27:210 [pid=75524 tid=75955] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:11:27:210 [pid=75531 tid=75958] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:11:27:210 [pid=75528 tid=75956] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:11:27:210 [pid=75533 tid=75957] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:11:27:210 [pid=75518 tid=75959] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:18:29:619 [pid=80338 tid=80768] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:18:29:620 [pid=80344 tid=80769] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:18:29:621 [pid=80328 tid=80770] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:18:29:621 [pid=80332 tid=80771] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:18:29:621 [pid=80341 tid=80773] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:18:29:621 [pid=80337 tid=80772] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:18:29:621 [pid=80347 tid=80774] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:18:29:622 [pid=80351 tid=80775] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:27:34:86 [pid=4112 tid=4553] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:27:34:89 [pid=4127 tid=4554] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:27:34:90 [pid=4116 tid=4555] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:27:34:90 [pid=4109 tid=4556] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:27:34:90 [pid=4124 tid=4557] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:27:34:91 [pid=4105 tid=4558] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:27:34:91 [pid=4131 tid=4559] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:27:34:92 [pid=4121 tid=4560] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:31:53:966 [pid=6884 tid=7307] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:31:53:968 [pid=6864 tid=7308] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:31:53:968 [pid=6877 tid=7309] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:31:53:969 [pid=6880 tid=7310] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:31:53:969 [pid=6871 tid=7311] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:31:53:969 [pid=6875 tid=7313] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:31:53:970 [pid=6887 tid=7312] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:31:53:970 [pid=6868 tid=7314] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:35:10:106 [pid=9175 tid=9623] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:35:10:106 [pid=9184 tid=9622] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:35:10:106 [pid=9182 tid=9624] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:35:10:106 [pid=9169 tid=9621] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:35:10:107 [pid=9187 tid=9625] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:35:10:107 [pid=9174 tid=9626] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:35:10:108 [pid=9177 tid=9627] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:35:10:108 [pid=9165 tid=9628] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:42:31:385 [pid=13401 tid=13835] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:42:31:391 [pid=13383 tid=13836] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:42:31:393 [pid=13395 tid=13838] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:42:31:393 [pid=13397 tid=13839] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:42:31:393 [pid=13375 tid=13840] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:42:31:394 [pid=13378 tid=13842] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:42:31:394 [pid=13390 tid=13837] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:42:31:394 [pid=13392 tid=13841] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:44:53:527 [pid=15429 tid=15860] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:44:53:527 [pid=15435 tid=15859] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:44:53:528 [pid=15423 tid=15862] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:44:53:528 [pid=15442 tid=15861] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:44:53:529 [pid=15428 tid=15866] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:44:53:529 [pid=15438 tid=15863] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:44:53:529 [pid=15419 tid=15864] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:44:53:529 [pid=15432 tid=15865] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:45:36:249 [pid=16173 tid=16581] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:45:36:249 [pid=16168 tid=16580] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:45:36:249 [pid=16177 tid=16585] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:45:36:249 [pid=16165 tid=16584] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:45:36:249 [pid=16159 tid=16583] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:45:36:250 [pid=16162 tid=16582] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:45:36:250 [pid=16185 tid=16586] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:45:36:250 [pid=16181 tid=16587] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:46:45:31 [pid=17124 tid=17559] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:46:45:31 [pid=17126 tid=17556] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:46:45:31 [pid=17133 tid=17557] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:46:45:31 [pid=17120 tid=17558] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:46:45:31 [pid=17115 tid=17560] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:46:45:32 [pid=17139 tid=17561] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:46:45:32 [pid=17130 tid=17563] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:46:45:33 [pid=17135 tid=17562] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:53:55:452 [pid=22212 tid=22643] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:53:55:453 [pid=22210 tid=22645] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:53:55:454 [pid=22222 tid=22644] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:53:55:454 [pid=22204 tid=22646] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:53:55:454 [pid=22200 tid=22647] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:53:55:455 [pid=22209 tid=22648] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:53:55:455 [pid=22216 tid=22650] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 03:53:55:456 [pid=22219 tid=22649] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded diff --git a/testing/dask-worker-space/global.lock b/testing/dask-worker-space/global.lock new file mode 100644 index 00000000000..e69de29bb2d diff --git a/testing/dask-worker-space/purge.lock b/testing/dask-worker-space/purge.lock new file mode 100644 index 00000000000..e69de29bb2d diff --git a/testing/testing2.ipynb b/testing/testing2.ipynb new file mode 100644 index 00000000000..7e2109aba78 --- /dev/null +++ b/testing/testing2.ipynb @@ -0,0 +1,561 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2022-05-20 03:53:39,266 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", + "2022-05-20 03:53:39,313 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", + "2022-05-20 03:53:39,314 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", + "2022-05-20 03:53:39,323 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", + "2022-05-20 03:53:39,323 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", + "2022-05-20 03:53:39,362 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", + "2022-05-20 03:53:39,395 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", + "2022-05-20 03:53:39,420 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", + "/opt/conda/envs/rapids/lib/python3.8/site-packages/cudf/core/dataframe.py:1055: FutureWarning: The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning.\n", + " return pd.Series(self._dtypes)\n", + "/opt/conda/envs/rapids/lib/python3.8/site-packages/cudf/core/indexed_frame.py:2328: FutureWarning: append is deprecated and will be removed in a future version. Use concat instead.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'resource_handle': , 'mg': , 'sources': Series([], Name: global_id, dtype: int32), 'weights': '0 1.0\\n1 1.0\\n2 1.0\\n3 1.0\\n4 1.0\\n5 1.0\\n6 1.0\\n7 1.0\\n8 1.0\\n9 1.0\\n10 1.0\\n11 1.0\\n12 1.0\\n13 1.0\\n14 1.0\\nName: renumbered_src, dtype: float32', 'direction_optimizing': False, 'depth_limit': 0, 'return_predecessors': True, 'do_expensive_check': True, 'data': [ renumbered_src renumbered_dst\n", + "0 6 18\n", + "1 5 18\n", + "2 1 18\n", + "3 13 18\n", + "4 13 19\n", + "5 13 21\n", + "6 13 22\n", + "7 14 18\n", + "8 17 18\n", + "9 30 18\n", + "10 30 22\n", + "11 31 19\n", + "12 31 18\n", + "13 32 23\n", + "14 31 22]}\n", + "{'resource_handle': , 'mg': , 'sources': Series([], Name: global_id, dtype: int32), 'weights': '0 1.0\\n1 1.0\\n2 1.0\\n3 1.0\\n4 1.0\\n5 1.0\\n6 1.0\\n7 1.0\\n8 1.0\\n9 1.0\\n10 1.0\\n11 1.0\\nName: renumbered_src, dtype: float32', 'direction_optimizing': False, 'depth_limit': 0, 'return_predecessors': True, 'do_expensive_check': True, 'data': [ renumbered_src renumbered_dst\n", + "0 8 18\n", + "1 11 18\n", + "2 19 21\n", + "3 19 18\n", + "4 21 19\n", + "5 18 19\n", + "6 26 20\n", + "7 28 18\n", + "8 25 23\n", + "9 27 22\n", + "10 24 23\n", + "11 25 20]}{'resource_handle': , 'mg': , 'sources': Series([], Name: global_id, dtype: int32), 'weights': '0 1.0\\n1 1.0\\n2 1.0\\n3 1.0\\n4 1.0\\n5 1.0\\n6 1.0\\n7 1.0\\n8 1.0\\n9 1.0\\n10 1.0\\n11 1.0\\n12 1.0\\n13 1.0\\nName: renumbered_src, dtype: float32', 'direction_optimizing': False, 'depth_limit': 0, 'return_predecessors': True, 'do_expensive_check': True, 'data': [ renumbered_src renumbered_dst\n", + "0 7 0\n", + "1 7 2\n", + "2 7 3\n", + "3 7 4\n", + "4 8 1\n", + "5 10 0\n", + "6 18 5\n", + "7 18 1\n", + "8 18 6\n", + "9 24 3\n", + "10 24 2\n", + "11 24 4\n", + "12 24 0\n", + "13 24 1]}\n", + "\n", + "{'resource_handle': , 'mg': , 'sources': Series([], Name: global_id, dtype: int32), 'weights': '0 1.0\\n1 1.0\\n2 1.0\\n3 1.0\\n4 1.0\\n5 1.0\\n6 1.0\\n7 1.0\\n8 1.0\\n9 1.0\\n10 1.0\\n11 1.0\\n12 1.0\\nName: renumbered_src, dtype: float32', 'direction_optimizing': False, 'depth_limit': 0, 'return_predecessors': True, 'do_expensive_check': True, 'data': [ renumbered_src renumbered_dst\n", + "0 2 0\n", + "1 3 0\n", + "2 0 3\n", + "3 0 2\n", + "4 13 4\n", + "5 13 6\n", + "6 13 1\n", + "7 13 2\n", + "8 13 5\n", + "9 30 1\n", + "10 30 3\n", + "11 30 2\n", + "12 30 0]}\n", + "{'resource_handle': , 'mg': , 'sources': Series([], Name: global_id, dtype: int32), 'weights': '0 1.0\\n1 1.0\\n2 1.0\\n3 1.0\\n4 1.0\\n5 1.0\\n6 1.0\\n7 1.0\\n8 1.0\\n9 1.0\\n10 1.0\\n11 1.0\\n12 1.0\\n13 1.0\\n14 1.0\\n15 1.0\\n16 1.0\\n17 1.0\\n18 1.0\\n19 1.0\\n20 1.0\\n21 1.0\\n22 1.0\\n23 1.0\\n24 1.0\\nName: renumbered_src, dtype: float32', 'direction_optimizing': False, 'depth_limit': 0, 'return_predecessors': True, 'do_expensive_check': True, 'data': [ renumbered_src renumbered_dst\n", + "0 3 30\n", + "1 0 30\n", + "2 1 30\n", + "3 1 24\n", + "4 0 24\n", + "5 3 24\n", + "6 2 30\n", + "7 2 24\n", + "8 4 24\n", + "9 14 24\n", + "10 14 27\n", + "11 13 28\n", + "12 13 31\n", + "13 15 30\n", + "14 16 30\n", + "15 22 27\n", + "16 23 24\n", + "17 23 25\n", + "18 23 32\n", + "19 22 31\n", + "20 22 30\n", + "21 33 24\n", + "22 30 24\n", + "23 32 24\n", + "24 32 26]}\n", + "{'resource_handle': , 'mg': , 'sources': 0 7\n", + "Name: global_id, dtype: int32, 'weights': '0 1.0\\n1 1.0\\n2 1.0\\n3 1.0\\n4 1.0\\n5 1.0\\n6 1.0\\n7 1.0\\n8 1.0\\n9 1.0\\n10 1.0\\n11 1.0\\n12 1.0\\n13 1.0\\n14 1.0\\n15 1.0\\n16 1.0\\n17 1.0\\n18 1.0\\n19 1.0\\nName: renumbered_src, dtype: float32', 'direction_optimizing': False, 'depth_limit': 0, 'return_predecessors': True, 'do_expensive_check': True, 'data': [ renumbered_src renumbered_dst\n", + "0 7 8\n", + "1 8 13\n", + "2 8 7\n", + "3 9 14\n", + "4 11 13\n", + "5 19 13\n", + "6 18 13\n", + "7 18 14\n", + "8 21 13\n", + "9 18 17\n", + "10 18 8\n", + "11 18 11\n", + "12 28 13\n", + "13 29 7\n", + "14 27 14\n", + "15 24 14\n", + "16 27 9\n", + "17 24 7\n", + "18 24 12\n", + "19 24 10]}\n", + "{'resource_handle': , 'mg': , 'sources': Series([], Name: global_id, dtype: int32), 'weights': '0 1.0\\n1 1.0\\n2 1.0\\n3 1.0\\n4 1.0\\n5 1.0\\n6 1.0\\n7 1.0\\n8 1.0\\n9 1.0\\n10 1.0\\n11 1.0\\n12 1.0\\n13 1.0\\n14 1.0\\n15 1.0\\n16 1.0\\n17 1.0\\n18 1.0\\n19 1.0\\n20 1.0\\n21 1.0\\n22 1.0\\n23 1.0\\n24 1.0\\n25 1.0\\nName: renumbered_src, dtype: float32', 'direction_optimizing': False, 'depth_limit': 0, 'return_predecessors': True, 'do_expensive_check': True, 'data': [ renumbered_src renumbered_dst\n", + "0 12 24\n", + "1 9 31\n", + "2 9 27\n", + "3 10 24\n", + "4 7 33\n", + "5 7 30\n", + "6 7 29\n", + "7 7 24\n", + "8 18 30\n", + "9 18 31\n", + "10 18 28\n", + "11 20 25\n", + "12 20 26\n", + "13 19 31\n", + "14 29 24\n", + "15 24 32\n", + "16 24 33\n", + "17 24 30\n", + "18 24 29\n", + "19 25 26\n", + "20 24 26\n", + "21 25 24\n", + "22 24 25\n", + "23 26 32\n", + "24 26 24\n", + "25 26 25]}\n", + "{'resource_handle': , 'mg': , 'sources': Series([], Name: global_id, dtype: int32), 'weights': '0 1.0\\n1 1.0\\n2 1.0\\n3 1.0\\n4 1.0\\n5 1.0\\n6 1.0\\n7 1.0\\n8 1.0\\n9 1.0\\n10 1.0\\n11 1.0\\n12 1.0\\n13 1.0\\n14 1.0\\n15 1.0\\n16 1.0\\n17 1.0\\n18 1.0\\n19 1.0\\n20 1.0\\n21 1.0\\n22 1.0\\n23 1.0\\n24 1.0\\n25 1.0\\n26 1.0\\n27 1.0\\n28 1.0\\n29 1.0\\n30 1.0\\nName: renumbered_src, dtype: float32', 'direction_optimizing': False, 'depth_limit': 0, 'return_predecessors': True, 'do_expensive_check': True, 'data': [ renumbered_src renumbered_dst\n", + "0 4 13\n", + "1 2 13\n", + "2 4 7\n", + "3 2 7\n", + "4 6 13\n", + "5 1 13\n", + "6 5 13\n", + "7 0 10\n", + "8 0 7\n", + "9 1 8\n", + "10 3 7\n", + "11 15 14\n", + "12 15 13\n", + "13 16 13\n", + "14 14 9\n", + "15 14 13\n", + "16 17 13\n", + "17 14 15\n", + "18 13 8\n", + "19 13 11\n", + "20 13 16\n", + "21 13 15\n", + "22 13 17\n", + "23 13 14\n", + "24 22 13\n", + "25 30 7\n", + "26 30 16\n", + "27 30 15\n", + "28 31 9\n", + "29 31 13\n", + "30 33 7]}\n", + "wait 1\n", + "wait 2\n", + "wait 3\n", + "wait 4\n", + "wait 5\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
distancevertexpredecessor
0214748364710-1
1214748364714-1
2214748364733-1
3214748364732-1
401-1
5214748364729-1
6214748364716-1
721474836478-1
8214748364727-1
921474836475-1
10214748364722-1
11214748364711-1
12214748364731-1
13214748364723-1
14214748364725-1
15214748364715-1
1621474836476-1
1721474836474-1
1821474836479-1
19214748364724-1
20214748364712-1
21214748364728-1
22214748364718-1
23214748364720-1
24214748364726-1
01131
11191
2171
31301
4121
51171
61211
7131
8101
\n", + "
" + ], + "text/plain": [ + " distance vertex predecessor\n", + "0 2147483647 10 -1\n", + "1 2147483647 14 -1\n", + "2 2147483647 33 -1\n", + "3 2147483647 32 -1\n", + "4 0 1 -1\n", + "5 2147483647 29 -1\n", + "6 2147483647 16 -1\n", + "7 2147483647 8 -1\n", + "8 2147483647 27 -1\n", + "9 2147483647 5 -1\n", + "10 2147483647 22 -1\n", + "11 2147483647 11 -1\n", + "12 2147483647 31 -1\n", + "13 2147483647 23 -1\n", + "14 2147483647 25 -1\n", + "15 2147483647 15 -1\n", + "16 2147483647 6 -1\n", + "17 2147483647 4 -1\n", + "18 2147483647 9 -1\n", + "19 2147483647 24 -1\n", + "20 2147483647 12 -1\n", + "21 2147483647 28 -1\n", + "22 2147483647 18 -1\n", + "23 2147483647 20 -1\n", + "24 2147483647 26 -1\n", + "0 1 13 1\n", + "1 1 19 1\n", + "2 1 7 1\n", + "3 1 30 1\n", + "4 1 2 1\n", + "5 1 17 1\n", + "6 1 21 1\n", + "7 1 3 1\n", + "8 1 0 1" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from dask.distributed import Client, wait\n", + "from dask_cuda import LocalCUDACluster\n", + "from cugraph.dask.comms import comms\n", + "import cugraph.dask as dask_cugraph\n", + "import cugraph, dask_cudf\n", + "\n", + "input_data_path = '../datasets/karate.csv'\n", + "\n", + "cluster = LocalCUDACluster()\n", + "client = Client(cluster)\n", + "comms.initialize(p2p=True)\n", + "\n", + "# Helper function to set the reader chunk size to automatically get one partition per GPU\n", + "chunksize = dask_cugraph.get_chunksize(input_data_path)\n", + "\n", + "# Multi-GPU CSV reader\n", + "e_list = dask_cudf.read_csv(input_data_path,\n", + " chunksize = chunksize,\n", + " delimiter=' ',\n", + " names=['src', 'dst'],\n", + " dtype=['int32', 'int32'])\n", + "\n", + "G = cugraph.DiGraph()\n", + "G.from_dask_cudf_edgelist(e_list, source='src', destination='dst')\n", + "\n", + "# now run BFS\n", + "bfs_df = dask_cugraph.bfs(G, 1, depth_limit=0).compute()\n", + "\n", + "# All done, clean up\n", + "comms.destroy()\n", + "client.close()\n", + "cluster.close()\n", + "\n", + "bfs_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "comms.destroy()\n", + "client.close()\n", + "cluster.close()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "interpreter": { + "hash": "f708a36acfaef0acf74ccd43dfb58100269bf08fb79032a1e0a6f35bd9856f51" + }, + "kernelspec": { + "display_name": "Python 3.8.10 ('base')", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} From d2323abdd06debad79bc4e480af1d96c803fa47f Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 20 May 2022 14:07:25 +0000 Subject: [PATCH 08/14] Remove test files --- testing/cufile.log | 608 -------------------------- testing/dask-worker-space/global.lock | 0 testing/dask-worker-space/purge.lock | 0 testing/testing2.ipynb | 561 ------------------------ 4 files changed, 1169 deletions(-) delete mode 100644 testing/cufile.log delete mode 100644 testing/dask-worker-space/global.lock delete mode 100644 testing/dask-worker-space/purge.lock delete mode 100644 testing/testing2.ipynb diff --git a/testing/cufile.log b/testing/cufile.log deleted file mode 100644 index 17da161a231..00000000000 --- a/testing/cufile.log +++ /dev/null @@ -1,608 +0,0 @@ - 18-05-2022 20:33:46:285 [pid=29501 tid=29847] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:33:46:291 [pid=29508 tid=29848] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:33:46:291 [pid=29487 tid=29849] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:33:46:291 [pid=29504 tid=29852] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:33:46:291 [pid=29499 tid=29850] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:33:46:291 [pid=29492 tid=29853] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:33:46:292 [pid=29510 tid=29851] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:33:46:293 [pid=29496 tid=29854] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:40:53:260 [pid=33667 tid=34102] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:40:53:261 [pid=33673 tid=34103] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:40:53:262 [pid=33677 tid=34104] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:40:53:262 [pid=33655 tid=34108] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:40:53:263 [pid=33660 tid=34105] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:40:53:263 [pid=33668 tid=34106] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:40:53:263 [pid=33664 tid=34107] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:40:53:264 [pid=33672 tid=34109] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:56:51:316 [pid=43329 tid=43755] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:56:51:317 [pid=43320 tid=43757] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:56:51:317 [pid=43325 tid=43756] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:56:51:319 [pid=43313 tid=43760] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:56:51:319 [pid=43312 tid=43759] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:56:51:319 [pid=43303 tid=43758] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:56:51:320 [pid=43308 tid=43761] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 20:56:51:320 [pid=43318 tid=43762] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:03:31:46 [pid=48019 tid=48460] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:03:31:52 [pid=48028 tid=48463] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:03:31:52 [pid=48038 tid=48461] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:03:31:52 [pid=48041 tid=48465] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:03:31:52 [pid=48033 tid=48464] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:03:31:52 [pid=48034 tid=48466] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:03:31:52 [pid=48023 tid=48462] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:03:31:53 [pid=48030 tid=48467] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:09:28:313 [pid=52083 tid=52518] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:09:28:314 [pid=52094 tid=52520] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:09:28:315 [pid=52100 tid=52521] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:09:28:315 [pid=52091 tid=52524] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:09:28:315 [pid=52080 tid=52519] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:09:28:315 [pid=52097 tid=52525] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:09:28:315 [pid=52088 tid=52522] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:09:28:315 [pid=52105 tid=52523] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:11:15:590 [pid=53538 tid=53962] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:11:15:591 [pid=53531 tid=53963] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:11:15:592 [pid=53530 tid=53966] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:11:15:592 [pid=53543 tid=53969] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:11:15:592 [pid=53521 tid=53965] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:11:15:592 [pid=53526 tid=53964] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:11:15:592 [pid=53532 tid=53967] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:11:15:593 [pid=53540 tid=53968] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:14:27:272 [pid=55658 tid=56095] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:14:27:273 [pid=55663 tid=56096] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:14:27:273 [pid=55668 tid=56097] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:14:27:273 [pid=55646 tid=56099] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:14:27:273 [pid=55649 tid=56098] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:14:27:275 [pid=55665 tid=56102] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:14:27:275 [pid=55657 tid=56101] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:14:27:275 [pid=55653 tid=56100] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:22:09:246 [pid=60251 tid=60694] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:22:09:247 [pid=60265 tid=60697] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:22:09:248 [pid=60263 tid=60698] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:22:09:248 [pid=60260 tid=60699] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:22:09:248 [pid=60254 tid=60696] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:22:09:248 [pid=60243 tid=60700] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:22:09:249 [pid=60256 tid=60695] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:22:09:249 [pid=60246 tid=60701] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:25:07:609 [pid=62455 tid=62899] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:25:07:611 [pid=62471 tid=62901] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:25:07:611 [pid=62464 tid=62900] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:25:07:611 [pid=62462 tid=62903] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:25:07:611 [pid=62459 tid=62902] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:25:07:612 [pid=62447 tid=62905] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:25:07:612 [pid=62453 tid=62904] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:25:07:613 [pid=62466 tid=62906] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:27:00:331 [pid=63933 tid=64348] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:27:00:331 [pid=63910 tid=64351] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:27:00:332 [pid=63926 tid=64349] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:27:00:332 [pid=63921 tid=64350] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:27:00:332 [pid=63913 tid=64352] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:27:00:333 [pid=63929 tid=64353] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:27:00:333 [pid=63923 tid=64355] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:27:00:333 [pid=63918 tid=64354] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:29:06:532 [pid=65525 tid=65954] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:29:06:533 [pid=65520 tid=65955] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:29:06:533 [pid=65517 tid=65956] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:29:06:533 [pid=65507 tid=65959] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:29:06:534 [pid=65511 tid=65957] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:29:06:534 [pid=65503 tid=65958] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:29:06:534 [pid=65513 tid=65960] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:29:06:535 [pid=65522 tid=65961] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:32:02:555 [pid=67503 tid=67940] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:32:02:556 [pid=67522 tid=67941] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:32:02:556 [pid=67507 tid=67942] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:32:02:558 [pid=67529 tid=67944] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:32:02:558 [pid=67516 tid=67943] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:32:02:558 [pid=67512 tid=67946] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:32:02:558 [pid=67520 tid=67945] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:32:02:559 [pid=67515 tid=67947] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:33:53:741 [pid=69000 tid=69450] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:33:53:742 [pid=69021 tid=69451] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:33:53:743 [pid=69010 tid=69452] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:33:53:743 [pid=69006 tid=69453] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:33:53:743 [pid=69019 tid=69454] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:33:53:744 [pid=69015 tid=69456] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:33:53:745 [pid=69012 tid=69455] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:33:53:745 [pid=69024 tid=69457] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:36:02:706 [pid=70640 tid=71056] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:36:02:708 [pid=70623 tid=71058] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:36:02:708 [pid=70618 tid=71057] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:36:02:709 [pid=70625 tid=71060] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:36:02:709 [pid=70635 tid=71059] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:36:02:709 [pid=70637 tid=71061] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:36:02:710 [pid=70632 tid=71062] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:36:02:710 [pid=70629 tid=71063] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:39:51:49 [pid=73079 tid=73499] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:39:51:49 [pid=73078 tid=73500] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:39:51:49 [pid=73057 tid=73502] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:39:51:50 [pid=73067 tid=73501] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:39:51:50 [pid=73075 tid=73503] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:39:51:51 [pid=73063 tid=73505] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:39:51:51 [pid=73073 tid=73504] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:39:51:51 [pid=73054 tid=73506] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:51:28:124 [pid=79799 tid=80239] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:51:28:126 [pid=79802 tid=80240] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:51:28:126 [pid=79791 tid=80241] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:51:28:127 [pid=79814 tid=80242] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:51:28:127 [pid=79804 tid=80243] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:51:28:127 [pid=79794 tid=80245] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:51:28:127 [pid=79811 tid=80244] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:51:28:128 [pid=79807 tid=80246] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:52:27:910 [pid=80826 tid=81257] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:52:27:910 [pid=80817 tid=81258] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:52:27:911 [pid=80824 tid=81260] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:52:27:912 [pid=80815 tid=81261] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:52:27:912 [pid=80809 tid=81259] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:52:27:912 [pid=80804 tid=81262] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:52:27:914 [pid=80819 tid=81264] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:52:27:914 [pid=80814 tid=81263] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:57:17:775 [pid=2164 tid=2606] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:57:17:775 [pid=2167 tid=2607] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:57:17:776 [pid=2153 tid=2609] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:57:17:777 [pid=2157 tid=2610] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:57:17:777 [pid=2172 tid=2611] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:57:17:778 [pid=2169 tid=2608] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:57:17:779 [pid=2159 tid=2612] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 21:57:17:780 [pid=2175 tid=2613] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:02:07:460 [pid=5167 tid=5606] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:02:07:463 [pid=5163 tid=5607] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:02:07:463 [pid=5173 tid=5610] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:02:07:464 [pid=5154 tid=5609] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:02:07:464 [pid=5162 tid=5612] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:02:07:464 [pid=5176 tid=5611] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:02:07:464 [pid=5171 tid=5608] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:02:07:464 [pid=5157 tid=5613] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:05:53:105 [pid=7772 tid=8211] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:05:53:108 [pid=7779 tid=8213] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:05:53:109 [pid=7785 tid=8212] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:05:53:109 [pid=7770 tid=8216] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:05:53:109 [pid=7767 tid=8214] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:05:53:109 [pid=7777 tid=8217] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:05:53:110 [pid=7763 tid=8215] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:05:53:112 [pid=7783 tid=8218] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:07:08:763 [pid=8938 tid=9387] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:07:08:763 [pid=8956 tid=9386] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:07:08:763 [pid=8941 tid=9391] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:07:08:763 [pid=8948 tid=9385] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:07:08:763 [pid=8952 tid=9390] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:07:08:764 [pid=8959 tid=9389] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:07:08:765 [pid=8951 tid=9388] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:07:08:766 [pid=8943 tid=9392] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:12:05:679 [pid=12245 tid=12684] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:12:05:680 [pid=12258 tid=12685] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:12:05:680 [pid=12249 tid=12687] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:12:05:681 [pid=12251 tid=12689] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:12:05:682 [pid=12267 tid=12690] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:12:05:682 [pid=12256 tid=12693] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:12:05:682 [pid=12261 tid=12691] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:12:05:682 [pid=12264 tid=12692] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:30:20:23 [pid=22054 tid=22470] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:30:20:24 [pid=22048 tid=22471] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:30:20:25 [pid=22033 tid=22472] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:30:20:25 [pid=22036 tid=22474] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:30:20:26 [pid=22042 tid=22473] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:30:20:26 [pid=22045 tid=22475] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:30:20:27 [pid=22029 tid=22476] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:30:20:27 [pid=22051 tid=22477] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:32:39:898 [pid=23591 tid=24054] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:32:39:900 [pid=23586 tid=24055] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:32:39:901 [pid=23578 tid=24056] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:32:39:901 [pid=23600 tid=24057] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:32:39:902 [pid=23587 tid=24059] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:32:39:902 [pid=23597 tid=24058] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:32:39:903 [pid=23594 tid=24060] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:32:39:907 [pid=23580 tid=24061] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:37:56:507 [pid=27108 tid=27544] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:37:56:508 [pid=27102 tid=27546] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:37:56:509 [pid=27095 tid=27548] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:37:56:509 [pid=27112 tid=27545] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:37:56:509 [pid=27115 tid=27549] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:37:56:509 [pid=27098 tid=27547] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:37:56:510 [pid=27117 tid=27550] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:37:56:510 [pid=27106 tid=27551] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:40:55:977 [pid=29238 tid=29675] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:40:55:977 [pid=29244 tid=29674] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:40:55:979 [pid=29229 tid=29676] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:40:55:979 [pid=29235 tid=29678] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:40:55:979 [pid=29223 tid=29677] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:40:55:980 [pid=29225 tid=29680] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:40:55:980 [pid=29242 tid=29681] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:40:55:980 [pid=29233 tid=29679] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:43:18:116 [pid=31082 tid=31523] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:43:18:118 [pid=31085 tid=31524] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:43:18:120 [pid=31089 tid=31525] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:43:18:120 [pid=31101 tid=31526] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:43:18:120 [pid=31078 tid=31528] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:43:18:121 [pid=31104 tid=31527] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:43:18:122 [pid=31092 tid=31529] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:43:18:122 [pid=31094 tid=31530] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:45:39:700 [pid=32813 tid=33240] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:45:39:701 [pid=32806 tid=33243] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:45:39:701 [pid=32824 tid=33241] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:45:39:701 [pid=32821 tid=33242] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:45:39:701 [pid=32817 tid=33244] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:45:39:701 [pid=32801 tid=33245] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:45:39:701 [pid=32811 tid=33246] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:45:39:701 [pid=32818 tid=33247] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:47:30:634 [pid=34277 tid=34707] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:47:30:636 [pid=34273 tid=34709] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:47:30:636 [pid=34263 tid=34708] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:47:30:636 [pid=34278 tid=34711] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:47:30:636 [pid=34281 tid=34710] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:47:30:636 [pid=34270 tid=34712] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:47:30:637 [pid=34267 tid=34713] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:47:30:637 [pid=34259 tid=34714] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:49:06:710 [pid=35613 tid=36047] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:49:06:710 [pid=35619 tid=36048] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:49:06:710 [pid=35630 tid=36049] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:49:06:710 [pid=35608 tid=36051] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:49:06:711 [pid=35621 tid=36050] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:49:06:711 [pid=35627 tid=36052] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:49:06:712 [pid=35617 tid=36053] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:49:06:713 [pid=35624 tid=36054] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:50:24:407 [pid=36788 tid=37214] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:50:24:408 [pid=36794 tid=37213] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:50:24:409 [pid=36780 tid=37215] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:50:24:409 [pid=36782 tid=37216] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:50:24:410 [pid=36771 tid=37218] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:50:24:410 [pid=36785 tid=37219] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:50:24:410 [pid=36775 tid=37217] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:50:24:410 [pid=36791 tid=37220] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:57:20:181 [pid=40857 tid=41302] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:57:20:181 [pid=40877 tid=41303] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:57:20:181 [pid=40854 tid=41301] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:57:20:181 [pid=40864 tid=41300] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:57:20:181 [pid=40873 tid=41306] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:57:20:181 [pid=40870 tid=41299] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:57:20:182 [pid=40867 tid=41305] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 22:57:20:182 [pid=40862 tid=41304] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:03:17:277 [pid=44777 tid=45205] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:03:17:280 [pid=44771 tid=45206] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:03:17:281 [pid=44764 tid=45208] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:03:17:281 [pid=44780 tid=45207] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:03:17:281 [pid=44783 tid=45210] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:03:17:282 [pid=44786 tid=45209] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:03:17:282 [pid=44768 tid=45211] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:03:17:283 [pid=44775 tid=45212] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:07:15:271 [pid=47725 tid=48156] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:07:15:274 [pid=47734 tid=48157] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:07:15:274 [pid=47712 tid=48159] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:07:15:275 [pid=47719 tid=48161] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:07:15:275 [pid=47716 tid=48160] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:07:15:275 [pid=47729 tid=48158] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:07:15:275 [pid=47731 tid=48162] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:07:15:276 [pid=47723 tid=48163] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:11:22:946 [pid=50710 tid=51128] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:11:22:946 [pid=50706 tid=51127] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:11:22:946 [pid=50700 tid=51129] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:11:22:948 [pid=50689 tid=51130] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:11:22:949 [pid=50697 tid=51131] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:11:22:949 [pid=50709 tid=51132] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:11:22:950 [pid=50692 tid=51133] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:11:22:950 [pid=50703 tid=51134] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:14:06:999 [pid=52412 tid=52877] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:14:07:0 [pid=52418 tid=52880] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:14:07:1 [pid=52428 tid=52878] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:14:07:1 [pid=52417 tid=52879] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:14:07:1 [pid=52425 tid=52881] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:14:07:2 [pid=52409 tid=52883] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:14:07:3 [pid=52423 tid=52884] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:14:07:3 [pid=52431 tid=52882] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:17:55:635 [pid=54901 tid=55346] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:17:55:635 [pid=54895 tid=55347] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:17:55:637 [pid=54903 tid=55349] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:17:55:638 [pid=54906 tid=55348] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:17:55:638 [pid=54884 tid=55350] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:17:55:639 [pid=54887 tid=55351] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:17:55:639 [pid=54898 tid=55352] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:17:55:639 [pid=54893 tid=55353] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:26:04:858 [pid=59631 tid=60094] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:26:04:858 [pid=59626 tid=60092] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:26:04:858 [pid=59624 tid=60091] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:26:04:858 [pid=59639 tid=60093] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:26:04:858 [pid=59622 tid=60095] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:26:04:859 [pid=59637 tid=60096] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:26:04:860 [pid=59634 tid=60097] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:26:04:860 [pid=59616 tid=60098] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:27:22:787 [pid=60784 tid=61259] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:27:22:793 [pid=60780 tid=61260] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:27:22:793 [pid=60792 tid=61261] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:27:22:794 [pid=60800 tid=61262] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:27:22:795 [pid=60789 tid=61265] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:27:22:795 [pid=60797 tid=61263] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:27:22:795 [pid=60793 tid=61264] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:27:22:796 [pid=60803 tid=61266] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:29:11:578 [pid=62080 tid=62542] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:29:11:580 [pid=62083 tid=62543] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:29:11:580 [pid=62067 tid=62544] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:29:11:581 [pid=62075 tid=62545] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:29:11:582 [pid=62071 tid=62546] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:29:11:582 [pid=62089 tid=62547] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:29:11:582 [pid=62077 tid=62548] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:29:11:583 [pid=62086 tid=62549] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:30:23:221 [pid=63201 tid=63657] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:30:23:222 [pid=63183 tid=63658] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:30:23:222 [pid=63198 tid=63659] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:30:23:223 [pid=63190 tid=63661] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:30:23:223 [pid=63193 tid=63660] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:30:23:224 [pid=63179 tid=63662] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:30:23:224 [pid=63195 tid=63664] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:30:23:224 [pid=63187 tid=63663] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:32:30:548 [pid=65149 tid=65618] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:32:30:549 [pid=65143 tid=65619] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:32:30:550 [pid=65139 tid=65621] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:32:30:550 [pid=65157 tid=65620] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:32:30:550 [pid=65148 tid=65622] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:32:30:551 [pid=65164 tid=65623] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:32:30:551 [pid=65167 tid=65625] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:32:30:552 [pid=65152 tid=65624] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:38:23:257 [pid=68756 tid=69212] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:38:23:260 [pid=68754 tid=69213] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:38:23:260 [pid=68734 tid=69214] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:38:23:261 [pid=68751 tid=69215] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:38:23:261 [pid=68747 tid=69217] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:38:23:262 [pid=68742 tid=69218] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:38:23:262 [pid=68744 tid=69216] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:38:23:262 [pid=68738 tid=69219] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:40:15:510 [pid=70220 tid=70674] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:40:15:511 [pid=70214 tid=70675] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:40:15:512 [pid=70218 tid=70676] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:40:15:513 [pid=70208 tid=70679] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:40:15:513 [pid=70211 tid=70677] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:40:15:513 [pid=70195 tid=70678] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:40:15:514 [pid=70199 tid=70680] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:40:15:514 [pid=70206 tid=70681] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:44:38:239 [pid=72965 tid=73418] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:44:38:241 [pid=72957 tid=73419] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:44:38:242 [pid=72959 tid=73423] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:44:38:242 [pid=72968 tid=73424] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:44:38:242 [pid=72943 tid=73420] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:44:38:242 [pid=72962 tid=73421] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:44:38:242 [pid=72951 tid=73422] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:44:38:243 [pid=72947 tid=73425] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:46:03:522 [pid=74182 tid=74647] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:46:03:523 [pid=74175 tid=74648] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:46:03:525 [pid=74180 tid=74649] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:46:03:525 [pid=74192 tid=74650] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:46:03:525 [pid=74197 tid=74652] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:46:03:526 [pid=74195 tid=74651] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:46:03:526 [pid=74190 tid=74653] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:46:03:526 [pid=74187 tid=74654] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:57:00:611 [pid=80513 tid=80975] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:57:00:611 [pid=80523 tid=80976] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:57:00:612 [pid=80528 tid=80977] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:57:00:612 [pid=80518 tid=80978] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:57:00:613 [pid=80505 tid=80980] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:57:00:613 [pid=80510 tid=80979] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:57:00:614 [pid=80525 tid=80981] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:57:00:614 [pid=80519 tid=80982] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:58:57:597 [pid=544 tid=1005] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:58:57:597 [pid=533 tid=1006] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:58:57:599 [pid=555 tid=1007] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:58:57:599 [pid=553 tid=1010] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:58:57:599 [pid=547 tid=1008] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:58:57:600 [pid=558 tid=1009] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:58:57:600 [pid=542 tid=1011] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 18-05-2022 23:58:57:601 [pid=538 tid=1012] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:00:39:54 [pid=1930 tid=2392] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:00:39:54 [pid=1933 tid=2393] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:00:39:55 [pid=1919 tid=2394] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:00:39:55 [pid=1911 tid=2395] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:00:39:56 [pid=1914 tid=2396] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:00:39:56 [pid=1927 tid=2397] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:00:39:56 [pid=1925 tid=2399] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:00:39:57 [pid=1922 tid=2398] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:14:40:782 [pid=9971 tid=10450] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:14:40:784 [pid=9980 tid=10451] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:14:40:784 [pid=9989 tid=10452] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:14:40:784 [pid=9987 tid=10453] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:14:40:785 [pid=9964 tid=10454] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:14:40:785 [pid=9977 tid=10455] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:14:40:785 [pid=9975 tid=10456] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:14:40:786 [pid=9983 tid=10457] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:22:11:27 [pid=14342 tid=14796] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:22:11:28 [pid=14351 tid=14797] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:22:11:30 [pid=14337 tid=14798] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:22:11:31 [pid=14338 tid=14799] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:22:11:31 [pid=14333 tid=14800] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:22:11:32 [pid=14348 tid=14801] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:22:11:32 [pid=14329 tid=14803] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:22:11:32 [pid=14346 tid=14802] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:34:44:989 [pid=21462 tid=21941] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:34:44:991 [pid=21468 tid=21942] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:34:44:991 [pid=21471 tid=21943] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:34:44:991 [pid=21482 tid=21944] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:34:44:992 [pid=21460 tid=21945] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:34:44:992 [pid=21474 tid=21947] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:34:44:992 [pid=21476 tid=21946] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:34:44:993 [pid=21479 tid=21948] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:38:18:686 [pid=23812 tid=24323] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:38:18:690 [pid=23824 tid=24328] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:38:18:690 [pid=23819 tid=24327] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:38:18:690 [pid=23804 tid=24324] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:38:18:690 [pid=23827 tid=24325] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:38:18:691 [pid=23815 tid=24329] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:38:18:691 [pid=23820 tid=24326] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:38:18:691 [pid=23830 tid=24330] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:41:36:123 [pid=26025 tid=26497] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:41:36:124 [pid=26047 tid=26498] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:41:36:124 [pid=26030 tid=26499] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:41:36:124 [pid=26033 tid=26501] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:41:36:124 [pid=26041 tid=26500] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:41:36:125 [pid=26037 tid=26504] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:41:36:125 [pid=26050 tid=26502] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:41:36:125 [pid=26045 tid=26503] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:44:51:492 [pid=28419 tid=28898] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:44:51:492 [pid=28433 tid=28899] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:44:51:492 [pid=28428 tid=28897] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:44:51:492 [pid=28441 tid=28900] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:44:51:492 [pid=28423 tid=28901] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:44:51:493 [pid=28438 tid=28902] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:44:51:493 [pid=28435 tid=28903] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:44:51:494 [pid=28431 tid=28904] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:46:10:782 [pid=29633 tid=30084] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:46:10:782 [pid=29641 tid=30085] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:46:10:782 [pid=29634 tid=30086] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:46:10:783 [pid=29625 tid=30087] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:46:10:783 [pid=29627 tid=30088] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:46:10:783 [pid=29621 tid=30089] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:46:10:784 [pid=29624 tid=30090] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:46:10:785 [pid=29615 tid=30091] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:49:04:630 [pid=31759 tid=32218] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:49:04:630 [pid=31738 tid=32222] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:49:04:631 [pid=31761 tid=32221] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:49:04:631 [pid=31756 tid=32220] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:49:04:631 [pid=31743 tid=32224] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:49:04:631 [pid=31747 tid=32219] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:49:04:631 [pid=31751 tid=32223] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:49:04:631 [pid=31749 tid=32225] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:53:43:730 [pid=34660 tid=35111] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:53:43:731 [pid=34646 tid=35112] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:53:43:732 [pid=34665 tid=35113] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:53:43:733 [pid=34668 tid=35114] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:53:43:733 [pid=34656 tid=35117] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:53:43:733 [pid=34650 tid=35118] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:53:43:733 [pid=34662 tid=35116] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:53:43:734 [pid=34653 tid=35115] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:54:39:910 [pid=35642 tid=36095] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:54:39:912 [pid=35627 tid=36096] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:54:39:914 [pid=35622 tid=36097] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:54:39:914 [pid=35639 tid=36099] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:54:39:914 [pid=35634 tid=36098] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:54:39:915 [pid=35637 tid=36101] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:54:39:915 [pid=35646 tid=36100] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 00:54:39:915 [pid=35648 tid=36102] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:03:53:568 [pid=40924 tid=41389] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:03:53:570 [pid=40930 tid=41391] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:03:53:570 [pid=40935 tid=41392] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:03:53:570 [pid=40913 tid=41390] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:03:53:571 [pid=40933 tid=41393] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:03:53:572 [pid=40917 tid=41395] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:03:53:572 [pid=40925 tid=41394] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:03:53:573 [pid=40921 tid=41396] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:05:11:767 [pid=42105 tid=42575] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:05:11:767 [pid=42116 tid=42577] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:05:11:767 [pid=42097 tid=42578] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:05:11:768 [pid=42113 tid=42576] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:05:11:768 [pid=42111 tid=42579] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:05:11:768 [pid=42108 tid=42580] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:05:11:769 [pid=42119 tid=42581] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:05:11:769 [pid=42102 tid=42582] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:18:43:662 [pid=49974 tid=50417] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:18:43:662 [pid=49967 tid=50418] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:18:43:663 [pid=49947 tid=50419] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:18:43:663 [pid=49970 tid=50421] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:18:43:664 [pid=49960 tid=50420] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:18:43:664 [pid=49951 tid=50424] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:18:43:664 [pid=49961 tid=50422] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:18:43:664 [pid=49965 tid=50423] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:29:06:700 [pid=56026 tid=56482] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:29:06:701 [pid=56000 tid=56484] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:29:06:702 [pid=56007 tid=56483] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:29:06:702 [pid=56017 tid=56486] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:29:06:702 [pid=56004 tid=56485] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:29:06:704 [pid=56010 tid=56487] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:29:06:704 [pid=56024 tid=56488] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:29:06:704 [pid=56021 tid=56489] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:31:40:61 [pid=57827 tid=58300] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:31:40:64 [pid=57853 tid=58302] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:31:40:64 [pid=57839 tid=58301] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:31:40:64 [pid=57851 tid=58305] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:31:40:64 [pid=57842 tid=58303] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:31:40:65 [pid=57848 tid=58304] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:31:40:65 [pid=57845 tid=58307] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 19-05-2022 01:31:40:65 [pid=57834 tid=58306] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:25:58:86 [pid=49269 tid=49709] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:25:58:86 [pid=49289 tid=49710] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:25:58:87 [pid=49292 tid=49712] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:25:58:88 [pid=49280 tid=49711] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:25:58:88 [pid=49273 tid=49714] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:25:58:88 [pid=49286 tid=49713] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:25:58:88 [pid=49277 tid=49716] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:25:58:88 [pid=49283 tid=49715] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:30:21:153 [pid=52226 tid=52650] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:30:21:154 [pid=52232 tid=52651] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:30:21:155 [pid=52221 tid=52653] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:30:21:156 [pid=52214 tid=52657] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:30:21:156 [pid=52210 tid=52654] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:30:21:156 [pid=52223 tid=52652] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:30:21:156 [pid=52229 tid=52655] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 02:30:21:156 [pid=52218 tid=52656] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:11:27:205 [pid=75511 tid=75952] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:11:27:208 [pid=75522 tid=75954] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:11:27:208 [pid=75515 tid=75953] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:11:27:210 [pid=75524 tid=75955] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:11:27:210 [pid=75531 tid=75958] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:11:27:210 [pid=75528 tid=75956] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:11:27:210 [pid=75533 tid=75957] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:11:27:210 [pid=75518 tid=75959] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:18:29:619 [pid=80338 tid=80768] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:18:29:620 [pid=80344 tid=80769] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:18:29:621 [pid=80328 tid=80770] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:18:29:621 [pid=80332 tid=80771] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:18:29:621 [pid=80341 tid=80773] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:18:29:621 [pid=80337 tid=80772] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:18:29:621 [pid=80347 tid=80774] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:18:29:622 [pid=80351 tid=80775] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:27:34:86 [pid=4112 tid=4553] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:27:34:89 [pid=4127 tid=4554] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:27:34:90 [pid=4116 tid=4555] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:27:34:90 [pid=4109 tid=4556] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:27:34:90 [pid=4124 tid=4557] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:27:34:91 [pid=4105 tid=4558] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:27:34:91 [pid=4131 tid=4559] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:27:34:92 [pid=4121 tid=4560] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:31:53:966 [pid=6884 tid=7307] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:31:53:968 [pid=6864 tid=7308] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:31:53:968 [pid=6877 tid=7309] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:31:53:969 [pid=6880 tid=7310] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:31:53:969 [pid=6871 tid=7311] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:31:53:969 [pid=6875 tid=7313] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:31:53:970 [pid=6887 tid=7312] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:31:53:970 [pid=6868 tid=7314] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:35:10:106 [pid=9175 tid=9623] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:35:10:106 [pid=9184 tid=9622] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:35:10:106 [pid=9182 tid=9624] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:35:10:106 [pid=9169 tid=9621] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:35:10:107 [pid=9187 tid=9625] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:35:10:107 [pid=9174 tid=9626] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:35:10:108 [pid=9177 tid=9627] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:35:10:108 [pid=9165 tid=9628] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:42:31:385 [pid=13401 tid=13835] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:42:31:391 [pid=13383 tid=13836] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:42:31:393 [pid=13395 tid=13838] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:42:31:393 [pid=13397 tid=13839] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:42:31:393 [pid=13375 tid=13840] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:42:31:394 [pid=13378 tid=13842] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:42:31:394 [pid=13390 tid=13837] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:42:31:394 [pid=13392 tid=13841] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:44:53:527 [pid=15429 tid=15860] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:44:53:527 [pid=15435 tid=15859] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:44:53:528 [pid=15423 tid=15862] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:44:53:528 [pid=15442 tid=15861] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:44:53:529 [pid=15428 tid=15866] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:44:53:529 [pid=15438 tid=15863] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:44:53:529 [pid=15419 tid=15864] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:44:53:529 [pid=15432 tid=15865] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:45:36:249 [pid=16173 tid=16581] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:45:36:249 [pid=16168 tid=16580] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:45:36:249 [pid=16177 tid=16585] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:45:36:249 [pid=16165 tid=16584] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:45:36:249 [pid=16159 tid=16583] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:45:36:250 [pid=16162 tid=16582] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:45:36:250 [pid=16185 tid=16586] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:45:36:250 [pid=16181 tid=16587] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:46:45:31 [pid=17124 tid=17559] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:46:45:31 [pid=17126 tid=17556] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:46:45:31 [pid=17133 tid=17557] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:46:45:31 [pid=17120 tid=17558] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:46:45:31 [pid=17115 tid=17560] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:46:45:32 [pid=17139 tid=17561] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:46:45:32 [pid=17130 tid=17563] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:46:45:33 [pid=17135 tid=17562] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:53:55:452 [pid=22212 tid=22643] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:53:55:453 [pid=22210 tid=22645] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:53:55:454 [pid=22222 tid=22644] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:53:55:454 [pid=22204 tid=22646] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:53:55:454 [pid=22200 tid=22647] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:53:55:455 [pid=22209 tid=22648] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:53:55:455 [pid=22216 tid=22650] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 03:53:55:456 [pid=22219 tid=22649] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded diff --git a/testing/dask-worker-space/global.lock b/testing/dask-worker-space/global.lock deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testing/dask-worker-space/purge.lock b/testing/dask-worker-space/purge.lock deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/testing/testing2.ipynb b/testing/testing2.ipynb deleted file mode 100644 index 7e2109aba78..00000000000 --- a/testing/testing2.ipynb +++ /dev/null @@ -1,561 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2022-05-20 03:53:39,266 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", - "2022-05-20 03:53:39,313 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", - "2022-05-20 03:53:39,314 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", - "2022-05-20 03:53:39,323 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", - "2022-05-20 03:53:39,323 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", - "2022-05-20 03:53:39,362 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", - "2022-05-20 03:53:39,395 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", - "2022-05-20 03:53:39,420 - distributed.preloading - INFO - Import preload module: dask_cuda.initialize\n", - "/opt/conda/envs/rapids/lib/python3.8/site-packages/cudf/core/dataframe.py:1055: FutureWarning: The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning.\n", - " return pd.Series(self._dtypes)\n", - "/opt/conda/envs/rapids/lib/python3.8/site-packages/cudf/core/indexed_frame.py:2328: FutureWarning: append is deprecated and will be removed in a future version. Use concat instead.\n", - " warnings.warn(\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'resource_handle': , 'mg': , 'sources': Series([], Name: global_id, dtype: int32), 'weights': '0 1.0\\n1 1.0\\n2 1.0\\n3 1.0\\n4 1.0\\n5 1.0\\n6 1.0\\n7 1.0\\n8 1.0\\n9 1.0\\n10 1.0\\n11 1.0\\n12 1.0\\n13 1.0\\n14 1.0\\nName: renumbered_src, dtype: float32', 'direction_optimizing': False, 'depth_limit': 0, 'return_predecessors': True, 'do_expensive_check': True, 'data': [ renumbered_src renumbered_dst\n", - "0 6 18\n", - "1 5 18\n", - "2 1 18\n", - "3 13 18\n", - "4 13 19\n", - "5 13 21\n", - "6 13 22\n", - "7 14 18\n", - "8 17 18\n", - "9 30 18\n", - "10 30 22\n", - "11 31 19\n", - "12 31 18\n", - "13 32 23\n", - "14 31 22]}\n", - "{'resource_handle': , 'mg': , 'sources': Series([], Name: global_id, dtype: int32), 'weights': '0 1.0\\n1 1.0\\n2 1.0\\n3 1.0\\n4 1.0\\n5 1.0\\n6 1.0\\n7 1.0\\n8 1.0\\n9 1.0\\n10 1.0\\n11 1.0\\nName: renumbered_src, dtype: float32', 'direction_optimizing': False, 'depth_limit': 0, 'return_predecessors': True, 'do_expensive_check': True, 'data': [ renumbered_src renumbered_dst\n", - "0 8 18\n", - "1 11 18\n", - "2 19 21\n", - "3 19 18\n", - "4 21 19\n", - "5 18 19\n", - "6 26 20\n", - "7 28 18\n", - "8 25 23\n", - "9 27 22\n", - "10 24 23\n", - "11 25 20]}{'resource_handle': , 'mg': , 'sources': Series([], Name: global_id, dtype: int32), 'weights': '0 1.0\\n1 1.0\\n2 1.0\\n3 1.0\\n4 1.0\\n5 1.0\\n6 1.0\\n7 1.0\\n8 1.0\\n9 1.0\\n10 1.0\\n11 1.0\\n12 1.0\\n13 1.0\\nName: renumbered_src, dtype: float32', 'direction_optimizing': False, 'depth_limit': 0, 'return_predecessors': True, 'do_expensive_check': True, 'data': [ renumbered_src renumbered_dst\n", - "0 7 0\n", - "1 7 2\n", - "2 7 3\n", - "3 7 4\n", - "4 8 1\n", - "5 10 0\n", - "6 18 5\n", - "7 18 1\n", - "8 18 6\n", - "9 24 3\n", - "10 24 2\n", - "11 24 4\n", - "12 24 0\n", - "13 24 1]}\n", - "\n", - "{'resource_handle': , 'mg': , 'sources': Series([], Name: global_id, dtype: int32), 'weights': '0 1.0\\n1 1.0\\n2 1.0\\n3 1.0\\n4 1.0\\n5 1.0\\n6 1.0\\n7 1.0\\n8 1.0\\n9 1.0\\n10 1.0\\n11 1.0\\n12 1.0\\nName: renumbered_src, dtype: float32', 'direction_optimizing': False, 'depth_limit': 0, 'return_predecessors': True, 'do_expensive_check': True, 'data': [ renumbered_src renumbered_dst\n", - "0 2 0\n", - "1 3 0\n", - "2 0 3\n", - "3 0 2\n", - "4 13 4\n", - "5 13 6\n", - "6 13 1\n", - "7 13 2\n", - "8 13 5\n", - "9 30 1\n", - "10 30 3\n", - "11 30 2\n", - "12 30 0]}\n", - "{'resource_handle': , 'mg': , 'sources': Series([], Name: global_id, dtype: int32), 'weights': '0 1.0\\n1 1.0\\n2 1.0\\n3 1.0\\n4 1.0\\n5 1.0\\n6 1.0\\n7 1.0\\n8 1.0\\n9 1.0\\n10 1.0\\n11 1.0\\n12 1.0\\n13 1.0\\n14 1.0\\n15 1.0\\n16 1.0\\n17 1.0\\n18 1.0\\n19 1.0\\n20 1.0\\n21 1.0\\n22 1.0\\n23 1.0\\n24 1.0\\nName: renumbered_src, dtype: float32', 'direction_optimizing': False, 'depth_limit': 0, 'return_predecessors': True, 'do_expensive_check': True, 'data': [ renumbered_src renumbered_dst\n", - "0 3 30\n", - "1 0 30\n", - "2 1 30\n", - "3 1 24\n", - "4 0 24\n", - "5 3 24\n", - "6 2 30\n", - "7 2 24\n", - "8 4 24\n", - "9 14 24\n", - "10 14 27\n", - "11 13 28\n", - "12 13 31\n", - "13 15 30\n", - "14 16 30\n", - "15 22 27\n", - "16 23 24\n", - "17 23 25\n", - "18 23 32\n", - "19 22 31\n", - "20 22 30\n", - "21 33 24\n", - "22 30 24\n", - "23 32 24\n", - "24 32 26]}\n", - "{'resource_handle': , 'mg': , 'sources': 0 7\n", - "Name: global_id, dtype: int32, 'weights': '0 1.0\\n1 1.0\\n2 1.0\\n3 1.0\\n4 1.0\\n5 1.0\\n6 1.0\\n7 1.0\\n8 1.0\\n9 1.0\\n10 1.0\\n11 1.0\\n12 1.0\\n13 1.0\\n14 1.0\\n15 1.0\\n16 1.0\\n17 1.0\\n18 1.0\\n19 1.0\\nName: renumbered_src, dtype: float32', 'direction_optimizing': False, 'depth_limit': 0, 'return_predecessors': True, 'do_expensive_check': True, 'data': [ renumbered_src renumbered_dst\n", - "0 7 8\n", - "1 8 13\n", - "2 8 7\n", - "3 9 14\n", - "4 11 13\n", - "5 19 13\n", - "6 18 13\n", - "7 18 14\n", - "8 21 13\n", - "9 18 17\n", - "10 18 8\n", - "11 18 11\n", - "12 28 13\n", - "13 29 7\n", - "14 27 14\n", - "15 24 14\n", - "16 27 9\n", - "17 24 7\n", - "18 24 12\n", - "19 24 10]}\n", - "{'resource_handle': , 'mg': , 'sources': Series([], Name: global_id, dtype: int32), 'weights': '0 1.0\\n1 1.0\\n2 1.0\\n3 1.0\\n4 1.0\\n5 1.0\\n6 1.0\\n7 1.0\\n8 1.0\\n9 1.0\\n10 1.0\\n11 1.0\\n12 1.0\\n13 1.0\\n14 1.0\\n15 1.0\\n16 1.0\\n17 1.0\\n18 1.0\\n19 1.0\\n20 1.0\\n21 1.0\\n22 1.0\\n23 1.0\\n24 1.0\\n25 1.0\\nName: renumbered_src, dtype: float32', 'direction_optimizing': False, 'depth_limit': 0, 'return_predecessors': True, 'do_expensive_check': True, 'data': [ renumbered_src renumbered_dst\n", - "0 12 24\n", - "1 9 31\n", - "2 9 27\n", - "3 10 24\n", - "4 7 33\n", - "5 7 30\n", - "6 7 29\n", - "7 7 24\n", - "8 18 30\n", - "9 18 31\n", - "10 18 28\n", - "11 20 25\n", - "12 20 26\n", - "13 19 31\n", - "14 29 24\n", - "15 24 32\n", - "16 24 33\n", - "17 24 30\n", - "18 24 29\n", - "19 25 26\n", - "20 24 26\n", - "21 25 24\n", - "22 24 25\n", - "23 26 32\n", - "24 26 24\n", - "25 26 25]}\n", - "{'resource_handle': , 'mg': , 'sources': Series([], Name: global_id, dtype: int32), 'weights': '0 1.0\\n1 1.0\\n2 1.0\\n3 1.0\\n4 1.0\\n5 1.0\\n6 1.0\\n7 1.0\\n8 1.0\\n9 1.0\\n10 1.0\\n11 1.0\\n12 1.0\\n13 1.0\\n14 1.0\\n15 1.0\\n16 1.0\\n17 1.0\\n18 1.0\\n19 1.0\\n20 1.0\\n21 1.0\\n22 1.0\\n23 1.0\\n24 1.0\\n25 1.0\\n26 1.0\\n27 1.0\\n28 1.0\\n29 1.0\\n30 1.0\\nName: renumbered_src, dtype: float32', 'direction_optimizing': False, 'depth_limit': 0, 'return_predecessors': True, 'do_expensive_check': True, 'data': [ renumbered_src renumbered_dst\n", - "0 4 13\n", - "1 2 13\n", - "2 4 7\n", - "3 2 7\n", - "4 6 13\n", - "5 1 13\n", - "6 5 13\n", - "7 0 10\n", - "8 0 7\n", - "9 1 8\n", - "10 3 7\n", - "11 15 14\n", - "12 15 13\n", - "13 16 13\n", - "14 14 9\n", - "15 14 13\n", - "16 17 13\n", - "17 14 15\n", - "18 13 8\n", - "19 13 11\n", - "20 13 16\n", - "21 13 15\n", - "22 13 17\n", - "23 13 14\n", - "24 22 13\n", - "25 30 7\n", - "26 30 16\n", - "27 30 15\n", - "28 31 9\n", - "29 31 13\n", - "30 33 7]}\n", - "wait 1\n", - "wait 2\n", - "wait 3\n", - "wait 4\n", - "wait 5\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
distancevertexpredecessor
0214748364710-1
1214748364714-1
2214748364733-1
3214748364732-1
401-1
5214748364729-1
6214748364716-1
721474836478-1
8214748364727-1
921474836475-1
10214748364722-1
11214748364711-1
12214748364731-1
13214748364723-1
14214748364725-1
15214748364715-1
1621474836476-1
1721474836474-1
1821474836479-1
19214748364724-1
20214748364712-1
21214748364728-1
22214748364718-1
23214748364720-1
24214748364726-1
01131
11191
2171
31301
4121
51171
61211
7131
8101
\n", - "
" - ], - "text/plain": [ - " distance vertex predecessor\n", - "0 2147483647 10 -1\n", - "1 2147483647 14 -1\n", - "2 2147483647 33 -1\n", - "3 2147483647 32 -1\n", - "4 0 1 -1\n", - "5 2147483647 29 -1\n", - "6 2147483647 16 -1\n", - "7 2147483647 8 -1\n", - "8 2147483647 27 -1\n", - "9 2147483647 5 -1\n", - "10 2147483647 22 -1\n", - "11 2147483647 11 -1\n", - "12 2147483647 31 -1\n", - "13 2147483647 23 -1\n", - "14 2147483647 25 -1\n", - "15 2147483647 15 -1\n", - "16 2147483647 6 -1\n", - "17 2147483647 4 -1\n", - "18 2147483647 9 -1\n", - "19 2147483647 24 -1\n", - "20 2147483647 12 -1\n", - "21 2147483647 28 -1\n", - "22 2147483647 18 -1\n", - "23 2147483647 20 -1\n", - "24 2147483647 26 -1\n", - "0 1 13 1\n", - "1 1 19 1\n", - "2 1 7 1\n", - "3 1 30 1\n", - "4 1 2 1\n", - "5 1 17 1\n", - "6 1 21 1\n", - "7 1 3 1\n", - "8 1 0 1" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from dask.distributed import Client, wait\n", - "from dask_cuda import LocalCUDACluster\n", - "from cugraph.dask.comms import comms\n", - "import cugraph.dask as dask_cugraph\n", - "import cugraph, dask_cudf\n", - "\n", - "input_data_path = '../datasets/karate.csv'\n", - "\n", - "cluster = LocalCUDACluster()\n", - "client = Client(cluster)\n", - "comms.initialize(p2p=True)\n", - "\n", - "# Helper function to set the reader chunk size to automatically get one partition per GPU\n", - "chunksize = dask_cugraph.get_chunksize(input_data_path)\n", - "\n", - "# Multi-GPU CSV reader\n", - "e_list = dask_cudf.read_csv(input_data_path,\n", - " chunksize = chunksize,\n", - " delimiter=' ',\n", - " names=['src', 'dst'],\n", - " dtype=['int32', 'int32'])\n", - "\n", - "G = cugraph.DiGraph()\n", - "G.from_dask_cudf_edgelist(e_list, source='src', destination='dst')\n", - "\n", - "# now run BFS\n", - "bfs_df = dask_cugraph.bfs(G, 1, depth_limit=0).compute()\n", - "\n", - "# All done, clean up\n", - "comms.destroy()\n", - "client.close()\n", - "cluster.close()\n", - "\n", - "bfs_df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "comms.destroy()\n", - "client.close()\n", - "cluster.close()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "interpreter": { - "hash": "f708a36acfaef0acf74ccd43dfb58100269bf08fb79032a1e0a6f35bd9856f51" - }, - "kernelspec": { - "display_name": "Python 3.8.10 ('base')", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} From e74353801467a45800c18f8bbb42c46624977df8 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 20 May 2022 14:08:53 +0000 Subject: [PATCH 09/14] add line at end of file --- python/pylibcugraph/pylibcugraph/bfs.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pylibcugraph/pylibcugraph/bfs.pyx b/python/pylibcugraph/pylibcugraph/bfs.pyx index 4cf32b1e536..6886e6b059a 100644 --- a/python/pylibcugraph/pylibcugraph/bfs.pyx +++ b/python/pylibcugraph/pylibcugraph/bfs.pyx @@ -197,4 +197,4 @@ def bfs(ResourceHandle handle, _GPUGraph graph, # deallocate the no-longer needed result struct cugraph_paths_result_free(result_ptr) - return (cupy_distances, cupy_predecessors, cupy_vertices) \ No newline at end of file + return (cupy_distances, cupy_predecessors, cupy_vertices) From a7eaefcb4df7db93c5a0de4dc8ca2337a388bfba Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 20 May 2022 14:21:47 +0000 Subject: [PATCH 10/14] fix style --- python/cugraph/cugraph/dask/traversal/bfs.py | 70 ++++++++++---------- python/cugraph/cugraph/traversal/bfs.py | 23 ++++--- 2 files changed, 50 insertions(+), 43 deletions(-) diff --git a/python/cugraph/cugraph/dask/traversal/bfs.py b/python/cugraph/cugraph/dask/traversal/bfs.py index f634ce7dcf0..e4ff32152db 100644 --- a/python/cugraph/cugraph/dask/traversal/bfs.py +++ b/python/cugraph/cugraph/dask/traversal/bfs.py @@ -19,7 +19,6 @@ bfs as pylibcugraph_bfs, ) -from collections.abc import Iterable from dask.distributed import wait, default_client from cugraph.dask.common.input_utils import (get_distributed_data, get_vertex_partition_offsets) @@ -27,18 +26,19 @@ import cudf import dask_cudf + def _call_plc_mg_bfs( - sID, - data, - sources, - depth_limit, - src_col_name, - dst_col_name, - graph_properties, - num_edges, - direction_optimizing=False, - do_expensive_check=False, - return_predecessors=True): + sID, + data, + sources, + depth_limit, + src_col_name, + dst_col_name, + graph_properties, + num_edges, + direction_optimizing=False, + do_expensive_check=False, + return_predecessors=True): comms_handle = Comms.get_handle(sID) resource_handle = ResourceHandle(comms_handle.getHandle()) @@ -49,14 +49,14 @@ def _call_plc_mg_bfs( else cudf.Series((srcs + 1) / (srcs + 1), dtype='float32') mg = MGGraph( - resource_handle = resource_handle, - graph_properties = graph_properties, - src_array = srcs, - dst_array = dsts, - weight_array = weights, - store_transposed = False, - num_edges = num_edges, - do_expensive_check = do_expensive_check + resource_handle=resource_handle, + graph_properties=graph_properties, + src_array=srcs, + dst_array=dsts, + weight_array=weights, + store_transposed=False, + num_edges=num_edges, + do_expensive_check=do_expensive_check ) res = \ @@ -69,9 +69,10 @@ def _call_plc_mg_bfs( return_predecessors, True ) - + return res + def convert_to_cudf(cp_arrays): """ create a cudf DataFrame from cupy arrays @@ -83,6 +84,7 @@ def convert_to_cudf(cp_arrays): df["predecessor"] = cupy_predecessors return df + def bfs(input_graph, start, depth_limit=None, @@ -145,7 +147,6 @@ def bfs(input_graph, input_graph.compute_renumber_edge_list(transposed=False) ddf = input_graph.edgelist.edgelist_df vertex_partition_offsets = get_vertex_partition_offsets(input_graph) - num_verts = vertex_partition_offsets.iloc[-1] graph_properties = GraphProperties( is_multigraph=False) @@ -157,10 +158,9 @@ def bfs(input_graph, dst_col_name = input_graph.renumber_map.renumbered_dst_col_name renumber_ddf = input_graph.renumber_map.implementation.ddf col_names = input_graph.renumber_map.implementation.col_names - - if isinstance(start, - dask_cudf.DataFrame) or isinstance(start, - cudf.DataFrame): + + if isinstance(start, dask_cudf.DataFrame) \ + or isinstance(start, cudf.DataFrame): tmp_df = start tmp_col_names = start.columns else: @@ -180,14 +180,16 @@ def df_merge(df, tmp_df, tmp_col_names): x = df[0].merge(tmp_df, on=tmp_col_names, how='inner') return x['global_id'] - start = [client.submit(df_merge, - wf[1], - tmp_ddf, - col_names, - workers=[wf[0]]) - for idx, wf in enumerate(renumber_data.worker_to_parts.items() - ) - ] + start = [ + client.submit( + df_merge, + wf[1], + tmp_ddf, + col_names, + workers=[wf[0]] + ) + for idx, wf in enumerate(renumber_data.worker_to_parts.items()) + ] def count_src(df): return len(df) diff --git a/python/cugraph/cugraph/traversal/bfs.py b/python/cugraph/cugraph/traversal/bfs.py index b8f32eb315b..8d11fa9c66f 100644 --- a/python/cugraph/cugraph/traversal/bfs.py +++ b/python/cugraph/cugraph/traversal/bfs.py @@ -126,6 +126,7 @@ def _convert_df_to_output_type(df, input_type): else: raise TypeError(f"input type {input_type} is not a supported type.") + def _call_plc_bfs(G, sources, depth_limit, do_expensive_check=False, direction_optimizing=False, return_predecessors=True): handle = ResourceHandle() @@ -137,11 +138,11 @@ def _call_plc_bfs(G, sources, depth_limit, do_expensive_check=False, else cudf.Series((srcs + 1) / (srcs + 1), dtype='float32') sg = SGGraph( - resource_handle = handle, - graph_properties = GraphProperties(is_multigraph=G.is_multigraph()), - src_array = srcs, - dst_array = dsts, - weight_array = weights, + resource_handle=handle, + graph_properties=GraphProperties(is_multigraph=G.is_multigraph()), + src_array=srcs, + dst_array=dsts, + weight_array=weights, store_transposed=False, renumber=False, do_expensive_check=do_expensive_check @@ -149,7 +150,7 @@ def _call_plc_bfs(G, sources, depth_limit, do_expensive_check=False, distances, predecessors, vertices = \ pylibcugraph_bfs( - handle, + handle, sg, sources, direction_optimizing, @@ -157,13 +158,12 @@ def _call_plc_bfs(G, sources, depth_limit, do_expensive_check=False, return_predecessors, do_expensive_check ) - + return cudf.DataFrame({ 'distance': cudf.Series(distances), 'vertex': cudf.Series(vertices), 'predecessor': cudf.Series(predecessors), }) - def bfs(G, @@ -274,7 +274,12 @@ def bfs(G, else: start = cudf.Series(start, name='starts') - df = _call_plc_bfs(G, start, depth_limit, return_predecessors=return_predecessors) + df = _call_plc_bfs( + G, + start, + depth_limit, + return_predecessors=return_predecessors + ) if G.renumbered: df = G.unrenumber(df, "vertex") df = G.unrenumber(df, "predecessor") From 3c088092642ee382991b8f883126ea46eb090dd9 Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 20 May 2022 14:29:25 +0000 Subject: [PATCH 11/14] fix style --- python/cugraph/cugraph/dask/traversal/bfs.py | 4 +--- python/cugraph/cugraph/traversal/bfs.py | 8 ++++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/python/cugraph/cugraph/dask/traversal/bfs.py b/python/cugraph/cugraph/dask/traversal/bfs.py index e4ff32152db..947f0dc7eb5 100644 --- a/python/cugraph/cugraph/dask/traversal/bfs.py +++ b/python/cugraph/cugraph/dask/traversal/bfs.py @@ -20,8 +20,7 @@ ) from dask.distributed import wait, default_client -from cugraph.dask.common.input_utils import (get_distributed_data, - get_vertex_partition_offsets) +from cugraph.dask.common.input_utils import get_distributed_data import cugraph.dask.comms.comms as Comms import cudf import dask_cudf @@ -146,7 +145,6 @@ def bfs(input_graph, input_graph.compute_renumber_edge_list(transposed=False) ddf = input_graph.edgelist.edgelist_df - vertex_partition_offsets = get_vertex_partition_offsets(input_graph) graph_properties = GraphProperties( is_multigraph=False) diff --git a/python/cugraph/cugraph/traversal/bfs.py b/python/cugraph/cugraph/traversal/bfs.py index 8d11fa9c66f..4ef1b508047 100644 --- a/python/cugraph/cugraph/traversal/bfs.py +++ b/python/cugraph/cugraph/traversal/bfs.py @@ -128,7 +128,7 @@ def _convert_df_to_output_type(df, input_type): def _call_plc_bfs(G, sources, depth_limit, do_expensive_check=False, - direction_optimizing=False, return_predecessors=True): + direction_optimizing=False, return_predecessors=True): handle = ResourceHandle() srcs = G.edgelist.edgelist_df['src'] @@ -275,9 +275,9 @@ def bfs(G, start = cudf.Series(start, name='starts') df = _call_plc_bfs( - G, - start, - depth_limit, + G, + start, + depth_limit, return_predecessors=return_predecessors ) if G.renumbered: From 0651f13c7eb839bf292f5ba587c4e183e297b9ba Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 20 May 2022 17:10:43 +0000 Subject: [PATCH 12/14] remove experimental warning, fix build issue --- cpp/tests/CMakeLists.txt | 1 - python/cufile.log | 9 +++++++++ python/cugraph/cugraph/dask/traversal/bfs.py | 2 +- python/cugraph/cugraph/traversal/bfs.py | 2 +- .../pylibcugraph/pylibcugraph/experimental/__init__.py | 3 --- 5 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 python/cufile.log diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 3c71dfffd5c..15ff56acfb6 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -645,7 +645,6 @@ if(BUILD_CUGRAPH_MG_TESTS) ConfigureCTestMG(MG_CAPI_HITS c_api/mg_hits_test.c c_api/mg_test_utils.cpp) ConfigureCTestMG(MG_CAPI_UNIFORM_NEIGHBOR_SAMPLE c_api/mg_uniform_neighbor_sample_test.c c_api/mg_test_utils.cpp) ConfigureCTestMG(MG_CAPI_TRIANGLE_COUNT c_api/mg_triangle_count_test.c c_api/mg_test_utils.cpp) - ConfigureCTestMG(MG_CAPI_BFS c_api/mg_bfs_test.c c_api/mg_test_utils.cpp) else() message(FATAL_ERROR "OpenMPI NOT found, cannot build MG tests.") endif() diff --git a/python/cufile.log b/python/cufile.log new file mode 100644 index 00000000000..98d696a7de3 --- /dev/null +++ b/python/cufile.log @@ -0,0 +1,9 @@ + 20-05-2022 17:09:25:265 [pid=71536 tid=71536] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 17:09:25:461 [pid=71706 tid=72121] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 17:09:25:462 [pid=71701 tid=72122] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 17:09:25:464 [pid=71699 tid=72125] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 17:09:25:464 [pid=71705 tid=72124] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 17:09:25:464 [pid=71690 tid=72123] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 17:09:25:464 [pid=71713 tid=72126] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 17:09:25:464 [pid=71693 tid=72127] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded + 20-05-2022 17:09:25:465 [pid=71711 tid=72128] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded diff --git a/python/cugraph/cugraph/dask/traversal/bfs.py b/python/cugraph/cugraph/dask/traversal/bfs.py index 947f0dc7eb5..c7b847b4457 100644 --- a/python/cugraph/cugraph/dask/traversal/bfs.py +++ b/python/cugraph/cugraph/dask/traversal/bfs.py @@ -16,8 +16,8 @@ from pylibcugraph.experimental import (MGGraph, ResourceHandle, GraphProperties, - bfs as pylibcugraph_bfs, ) +from pylibcugraph import bfs as pylibcugraph_bfs from dask.distributed import wait, default_client from cugraph.dask.common.input_utils import get_distributed_data diff --git a/python/cugraph/cugraph/traversal/bfs.py b/python/cugraph/cugraph/traversal/bfs.py index 4ef1b508047..77fa02cad76 100644 --- a/python/cugraph/cugraph/traversal/bfs.py +++ b/python/cugraph/cugraph/traversal/bfs.py @@ -17,8 +17,8 @@ from pylibcugraph.experimental import (ResourceHandle, GraphProperties, SGGraph, - bfs as pylibcugraph_bfs, ) +from pylibcugraph import bfs as pylibcugraph_bfs from cugraph.structure.graph_classes import Graph, DiGraph from cugraph.utilities import (ensure_cugraph_obj, diff --git a/python/pylibcugraph/pylibcugraph/experimental/__init__.py b/python/pylibcugraph/pylibcugraph/experimental/__init__.py index c4b53e0229e..2587b2457a1 100644 --- a/python/pylibcugraph/pylibcugraph/experimental/__init__.py +++ b/python/pylibcugraph/pylibcugraph/experimental/__init__.py @@ -66,9 +66,6 @@ from pylibcugraph.node2vec import node2vec node2vec = promoted_experimental_warning_wrapper(node2vec) -from pylibcugraph.bfs import bfs -bfs = promoted_experimental_warning_wrapper(bfs) - from pylibcugraph.uniform_neighborhood_sampling import EXPERIMENTAL__uniform_neighborhood_sampling uniform_neighborhood_sampling = experimental_warning_wrapper(EXPERIMENTAL__uniform_neighborhood_sampling) From 9cac9edcc638713858933353c36fbf431531dc13 Mon Sep 17 00:00:00 2001 From: Alex Barghi <105237337+alexbarghi-nv@users.noreply.github.com> Date: Fri, 20 May 2022 14:11:10 -0400 Subject: [PATCH 13/14] Delete cufile.log --- python/cufile.log | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 python/cufile.log diff --git a/python/cufile.log b/python/cufile.log deleted file mode 100644 index 98d696a7de3..00000000000 --- a/python/cufile.log +++ /dev/null @@ -1,9 +0,0 @@ - 20-05-2022 17:09:25:265 [pid=71536 tid=71536] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 17:09:25:461 [pid=71706 tid=72121] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 17:09:25:462 [pid=71701 tid=72122] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 17:09:25:464 [pid=71699 tid=72125] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 17:09:25:464 [pid=71705 tid=72124] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 17:09:25:464 [pid=71690 tid=72123] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 17:09:25:464 [pid=71713 tid=72126] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 17:09:25:464 [pid=71693 tid=72127] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded - 20-05-2022 17:09:25:465 [pid=71711 tid=72128] ERROR cufio-drv:632 nvidia-fs.ko driver not loaded From 9f074a3d5579f1b2872037aa3f9afb79a135b4ce Mon Sep 17 00:00:00 2001 From: Alexandria Barghi Date: Fri, 20 May 2022 18:24:01 +0000 Subject: [PATCH 14/14] remove old wrappers --- .../cugraph/cugraph/dask/traversal/mg_bfs.pxd | 35 ----- .../cugraph/dask/traversal/mg_bfs_wrapper.pyx | 142 ------------------ 2 files changed, 177 deletions(-) delete mode 100644 python/cugraph/cugraph/dask/traversal/mg_bfs.pxd delete mode 100644 python/cugraph/cugraph/dask/traversal/mg_bfs_wrapper.pyx diff --git a/python/cugraph/cugraph/dask/traversal/mg_bfs.pxd b/python/cugraph/cugraph/dask/traversal/mg_bfs.pxd deleted file mode 100644 index 4434ae104ef..00000000000 --- a/python/cugraph/cugraph/dask/traversal/mg_bfs.pxd +++ /dev/null @@ -1,35 +0,0 @@ -# -# Copyright (c) 2020-2021, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from cugraph.structure.graph_utilities cimport * -from libcpp cimport bool - -cdef extern from "limits.h": - cdef int INT_MAX - cdef long LONG_MAX - -cdef extern from "cugraph/utilities/cython.hpp" namespace "cugraph::cython": - - cdef void call_bfs[vertex_t, weight_t]( - const handle_t &handle, - const graph_container_t &g, - vertex_t *identifiers, - vertex_t *distances, - vertex_t *predecessors, - vertex_t depth_limit, - const vertex_t *start_vertex, - size_t n_sources, - bool direction_optimizing) except + diff --git a/python/cugraph/cugraph/dask/traversal/mg_bfs_wrapper.pyx b/python/cugraph/cugraph/dask/traversal/mg_bfs_wrapper.pyx deleted file mode 100644 index 87f0eab1008..00000000000 --- a/python/cugraph/cugraph/dask/traversal/mg_bfs_wrapper.pyx +++ /dev/null @@ -1,142 +0,0 @@ -# -# Copyright (c) 2020-2022, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from cugraph.structure.utils_wrapper import * -from cugraph.dask.traversal cimport mg_bfs as c_bfs -import cudf -from cugraph.structure.graph_utilities cimport * -import cugraph.structure.graph_primtypes_wrapper as graph_primtypes_wrapper -from libc.stdint cimport uintptr_t - -def mg_bfs(input_df, - src_col_name, - dst_col_name, - num_global_verts, - num_global_edges, - vertex_partition_offsets, - rank, - handle, - segment_offsets, - start, - depth_limit, - return_distances=False): - """ - Call BFS - """ - cdef size_t handle_size_t = handle.getHandle() - handle_ = handle_size_t - - # Local COO information - src = input_df[src_col_name] - dst = input_df[dst_col_name] - vertex_t = src.dtype - if num_global_edges > (2**31 - 1): - edge_t = np.dtype("int64") - else: - edge_t = vertex_t - if "value" in input_df.columns: - weights = input_df['value'] - weight_t = weights.dtype - else: - weight_t = np.dtype("float32") - - - # FIXME: Offsets and indices are currently hardcoded to int, but this may - # not be acceptable in the future. - numberTypeMap = {np.dtype("int32") : numberTypeEnum.int32Type, - np.dtype("int64") : numberTypeEnum.int64Type, - np.dtype("float32") : numberTypeEnum.floatType, - np.dtype("double") : numberTypeEnum.doubleType} - - # FIXME: needs to be edge_t type not int - cdef int num_local_edges = len(src) - - cdef uintptr_t c_src_vertices = src.__cuda_array_interface__['data'][0] - cdef uintptr_t c_dst_vertices = dst.__cuda_array_interface__['data'][0] - cdef uintptr_t c_edge_weights = NULL - - # FIXME: data is on device, move to host (to_pandas()), convert to np array and access pointer to pass to C - vertex_partition_offsets_host = vertex_partition_offsets.values_host - cdef uintptr_t c_vertex_partition_offsets = vertex_partition_offsets_host.__array_interface__['data'][0] - - cdef vector[int] v_segment_offsets_32 - cdef vector[long] v_segment_offsets_64 - cdef uintptr_t c_segment_offsets - if (vertex_t == np.dtype("int32")): - v_segment_offsets_32 = segment_offsets - c_segment_offsets = v_segment_offsets_32.data() - else: - v_segment_offsets_64 = segment_offsets - c_segment_offsets = v_segment_offsets_64.data() - - cdef graph_container_t graph_container - - populate_graph_container(graph_container, - handle_[0], - c_src_vertices, c_dst_vertices, c_edge_weights, - c_vertex_partition_offsets, - c_segment_offsets, - len(segment_offsets) - 1, - ((numberTypeMap[vertex_t])), - ((numberTypeMap[edge_t])), - ((numberTypeMap[weight_t])), - num_local_edges, - num_global_verts, num_global_edges, - False, # BFS runs on unweighted graphs - False, - False, True) - - # Generate the cudf.DataFrame result - df = cudf.DataFrame() - df['vertex'] = cudf.Series(np.arange(vertex_partition_offsets.iloc[rank], vertex_partition_offsets.iloc[rank+1]), dtype=vertex_t) - df['predecessor'] = cudf.Series(np.zeros(len(df['vertex']), dtype=vertex_t)) - if (return_distances): - df['distance'] = cudf.Series(np.zeros(len(df['vertex']), dtype=vertex_t)) - - # Associate to cudf Series - cdef uintptr_t c_distance_ptr = NULL # Pointer to the DataFrame 'distance' Series - cdef uintptr_t c_predecessor_ptr = df['predecessor'].__cuda_array_interface__['data'][0] - if (return_distances): - c_distance_ptr = df['distance'].__cuda_array_interface__['data'][0] - cdef uintptr_t c_start_ptr = start.__cuda_array_interface__['data'][0] - - cdef bool direction_optimizing = 0 - - if vertex_t == np.int32: - if depth_limit is None: - depth_limit = c_bfs.INT_MAX - c_bfs.call_bfs[int, float](handle_[0], - graph_container, - NULL, - c_distance_ptr, - c_predecessor_ptr, - depth_limit, - c_start_ptr, - len(start), - direction_optimizing) - else: - if depth_limit is None: - depth_limit = c_bfs.LONG_MAX - c_bfs.call_bfs[long, float](handle_[0], - graph_container, - NULL, - c_distance_ptr, - c_predecessor_ptr, - depth_limit, - c_start_ptr, - len(start), - direction_optimizing) - return df