diff --git a/cpp/include/cugraph/utilities/cython.hpp b/cpp/include/cugraph/utilities/cython.hpp index 3a4f437bfd0..100a9d7db5e 100644 --- a/cpp/include/cugraph/utilities/cython.hpp +++ b/cpp/include/cugraph/utilities/cython.hpp @@ -524,6 +524,17 @@ void call_wcc(raft::handle_t const& handle, graph_container_t const& graph_container, vertex_t* components); +// Wrapper for calling HITS through a graph container +template +void call_hits(raft::handle_t const& handle, + graph_container_t const& graph_container, + weight_t* hubs, + weight_t* authorities, + size_t max_iter, + weight_t tolerance, + const weight_t* starting_value, + bool normalized); + // Wrapper for calling graph generator template std::unique_ptr call_generate_rmat_edgelist(raft::handle_t const& handle, diff --git a/cpp/src/utilities/cython.cu b/cpp/src/utilities/cython.cu index 2fe1b96b165..0f5ba693f28 100644 --- a/cpp/src/utilities/cython.cu +++ b/cpp/src/utilities/cython.cu @@ -1098,6 +1098,84 @@ void call_wcc(raft::handle_t const& handle, } } +// wrapper for HITS: +// +template +void call_hits(raft::handle_t const& handle, + graph_container_t const& graph_container, + weight_t* hubs, + weight_t* authorities, + size_t max_iter, + weight_t tolerance, + const weight_t* starting_value, + bool normalized) +{ + constexpr bool has_initial_hubs_guess{false}; + constexpr bool normalize{true}; + constexpr bool do_expensive_check{false}; + constexpr bool transposed{true}; + + // FIXME: most of these branches are not currently executed: MG support is not + // yet in the python API, and only int32_t edge types are being used. Consider + // removing these until actually needed. + + if (graph_container.is_multi_gpu) { + constexpr bool multi_gpu{true}; + if (graph_container.edgeType == numberTypeEnum::int32Type) { + auto graph = detail::create_graph( + handle, graph_container); + cugraph::hits(handle, + graph->view(), + reinterpret_cast(hubs), + reinterpret_cast(authorities), + tolerance, + max_iter, + has_initial_hubs_guess, + normalize, + do_expensive_check); + } else if (graph_container.edgeType == numberTypeEnum::int64Type) { + auto graph = detail::create_graph( + handle, graph_container); + cugraph::hits(handle, + graph->view(), + reinterpret_cast(hubs), + reinterpret_cast(authorities), + tolerance, + max_iter, + has_initial_hubs_guess, + normalize, + do_expensive_check); + } + } else { + constexpr bool multi_gpu{false}; + if (graph_container.edgeType == numberTypeEnum::int32Type) { + auto graph = detail::create_graph( + handle, graph_container); + cugraph::hits(handle, + graph->view(), + reinterpret_cast(hubs), + reinterpret_cast(authorities), + tolerance, + max_iter, + has_initial_hubs_guess, + normalize, + do_expensive_check); + } else if (graph_container.edgeType == numberTypeEnum::int64Type) { + auto graph = detail::create_graph( + handle, graph_container); + cugraph::hits(handle, + graph->view(), + reinterpret_cast(hubs), + reinterpret_cast(authorities), + tolerance, + max_iter, + has_initial_hubs_guess, + normalize, + do_expensive_check); + } + } +} + // wrapper for shuffling: // template @@ -1509,6 +1587,42 @@ template void call_wcc(raft::handle_t const& handle, graph_container_t const& graph_container, int64_t* components); +template void call_hits(raft::handle_t const& handle, + graph_container_t const& graph_container, + float* hubs, + float* authorities, + size_t max_iter, + float tolerance, + const float* starting_value, + bool normalized); + +template void call_hits(raft::handle_t const& handle, + graph_container_t const& graph_container, + double* hubs, + double* authorities, + size_t max_iter, + double tolerance, + const double* starting_value, + bool normalized); + +template void call_hits(raft::handle_t const& handle, + graph_container_t const& graph_container, + float* hubs, + float* authorities, + size_t max_iter, + float tolerance, + const float* starting_value, + bool normalized); + +template void call_hits(raft::handle_t const& handle, + graph_container_t const& graph_container, + double* hubs, + double* authorities, + size_t max_iter, + double tolerance, + const double* starting_value, + bool normalized); + template std::unique_ptr> call_shuffle( raft::handle_t const& handle, int32_t* edgelist_major_vertices, diff --git a/notebooks/link_analysis/HITS.ipynb b/notebooks/link_analysis/HITS.ipynb index 082b6c05185..01fd22929d5 100755 --- a/notebooks/link_analysis/HITS.ipynb +++ b/notebooks/link_analysis/HITS.ipynb @@ -5,7 +5,6 @@ "metadata": {}, "source": [ "# HITS\n", - "# Skip notebook test\n", "\n", "In this notebook, we will use both NetworkX and cuGraph to compute HITS. \n", "The NetworkX and cuGraph processes will be interleaved so that each step can be compared.\n", diff --git a/python/cugraph/cugraph/centrality/betweenness_centrality.py b/python/cugraph/cugraph/centrality/betweenness_centrality.py index 458f282dc2b..fb89d248280 100644 --- a/python/cugraph/cugraph/centrality/betweenness_centrality.py +++ b/python/cugraph/cugraph/centrality/betweenness_centrality.py @@ -48,7 +48,7 @@ def betweenness_centrality( Parameters ---------- G : cuGraph.Graph or networkx.Graph - The graph can be either directed (DiGraph) or undirected (Graph). + The graph can be either directed (Graph(directed=True)) or undirected. Weights in the graph are ignored, the current implementation uses BFS traversals. Use weight parameter if weights need to be considered (currently not supported) @@ -65,8 +65,8 @@ def betweenness_centrality( normalized : bool, optional Default is True. If true, the betweenness values are normalized by - __2 / ((n - 1) * (n - 2))__ for Graphs (undirected), and - __1 / ((n - 1) * (n - 2))__ for DiGraphs (directed graphs) + __2 / ((n - 1) * (n - 2))__ for undirected Graphs, and + __1 / ((n - 1) * (n - 2))__ for directed Graphs where n is the number of nodes in G. Normalization will ensure that values are in [0, 1], this normalization scales for the highest possible value where one @@ -170,7 +170,7 @@ def edge_betweenness_centrality( Parameters ---------- G : cuGraph.Graph or networkx.Graph - The graph can be either directed (DiGraph) or undirected (Graph). + The graph can be either directed (Graph(directed=True)) or undirected. Weights in the graph are ignored, the current implementation uses BFS traversals. Use weight parameter if weights need to be considered (currently not supported) @@ -186,8 +186,8 @@ def edge_betweenness_centrality( normalized : bool, optional Default is True. If true, the betweenness values are normalized by - 2 / (n * (n - 1)) for Graphs (undirected), and - 1 / (n * (n - 1)) for DiGraphs (directed graphs) + 2 / (n * (n - 1)) for undirected Graphs, and + 1 / (n * (n - 1)) for directed Graphs where n is the number of nodes in G. Normalization will ensure that values are in [0, 1], this normalization scales for the highest possible value where one diff --git a/python/cugraph/cugraph/centrality/betweenness_centrality_wrapper.pyx b/python/cugraph/cugraph/centrality/betweenness_centrality_wrapper.pyx index e63b6996816..4984f70822a 100644 --- a/python/cugraph/cugraph/centrality/betweenness_centrality_wrapper.pyx +++ b/python/cugraph/cugraph/centrality/betweenness_centrality_wrapper.pyx @@ -17,7 +17,6 @@ # cython: language_level = 3 from cugraph.centrality.betweenness_centrality cimport betweenness_centrality as c_betweenness_centrality -from cugraph.structure.graph_classes import DiGraph from cugraph.structure.graph_primtypes cimport * from libc.stdint cimport uintptr_t from libcpp cimport bool @@ -177,8 +176,7 @@ def batch_betweenness_centrality(input_graph, normalized, endpoints, comms = Comms.get_comms() replicated_adjlists = input_graph.batch_adjlists work_futures = [client.submit(run_mg_work, - (data, type(input_graph) - is DiGraph), + (data, input_graph.is_directed()), normalized, endpoints, weights, @@ -197,7 +195,7 @@ def sg_betweenness_centrality(input_graph, normalized, endpoints, weights, handle = Comms.get_default_handle() adjlist = input_graph.adjlist input_data = ((adjlist.offsets, adjlist.indices, adjlist.weights), - type(input_graph) is DiGraph) + input_graph.is_directed()) df = run_internal_work(handle, input_data, normalized, endpoints, weights, vertices, result_dtype) return df diff --git a/python/cugraph/cugraph/centrality/edge_betweenness_centrality_wrapper.pyx b/python/cugraph/cugraph/centrality/edge_betweenness_centrality_wrapper.pyx index 095d291c45e..f6c868de6e9 100644 --- a/python/cugraph/cugraph/centrality/edge_betweenness_centrality_wrapper.pyx +++ b/python/cugraph/cugraph/centrality/edge_betweenness_centrality_wrapper.pyx @@ -18,7 +18,6 @@ from cugraph.centrality.betweenness_centrality cimport edge_betweenness_centrality as c_edge_betweenness_centrality from cugraph.structure import graph_primtypes_wrapper -from cugraph.structure.graph_classes import DiGraph, Graph from cugraph.structure.graph_primtypes cimport * from libc.stdint cimport uintptr_t from libcpp cimport bool @@ -166,8 +165,7 @@ def batch_edge_betweenness_centrality(input_graph, comms = Comms.get_comms() replicated_adjlists = input_graph.batch_adjlists work_futures = [client.submit(run_mg_work, - (data, type(input_graph) - is DiGraph), + (data, input_graph.is_directed()), normalized, weights, vertices, @@ -188,7 +186,7 @@ def sg_edge_betweenness_centrality(input_graph, normalized, weights, handle = Comms.get_default_handle() adjlist = input_graph.adjlist input_data = ((adjlist.offsets, adjlist.indices, adjlist.weights), - type(input_graph) is DiGraph) + input_graph.is_directed()) df = run_internal_work(handle, input_data, normalized, weights, vertices, result_dtype) return df diff --git a/python/cugraph/cugraph/link_analysis/hits.pxd b/python/cugraph/cugraph/link_analysis/hits.pxd index 9e40f7444f9..c84f0c35e28 100644 --- a/python/cugraph/cugraph/link_analysis/hits.pxd +++ b/python/cugraph/cugraph/link_analysis/hits.pxd @@ -11,22 +11,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -# cython: profile=False -# distutils: language = c++ -# cython: embedsignature = True -# cython: language_level = 3 - -from cugraph.structure.graph_primtypes cimport * from libcpp cimport bool +from cugraph.structure.graph_utilities cimport graph_container_t +from cugraph.raft.common.handle cimport handle_t -cdef extern from "cugraph/algorithms.hpp" namespace "cugraph::gunrock": - cdef void hits[VT,ET,WT]( - const GraphCSRView[VT,ET,WT] &graph, +cdef extern from "cugraph/utilities/cython.hpp" namespace "cugraph::cython": + cdef void call_hits[vertex_t,weight_t]( + const handle_t &handle, + const graph_container_t &g, + weight_t *hubs, + weight_t *authorities, int max_iter, - WT tolerance, - const WT *starting_value, - bool normalized, - WT *hubs, - WT *authorities) except + + weight_t tolerance, + const weight_t *starting_value, + bool normalized) except + diff --git a/python/cugraph/cugraph/link_analysis/hits.py b/python/cugraph/cugraph/link_analysis/hits.py index 1dfb235fbd1..7176d1c66a1 100644 --- a/python/cugraph/cugraph/link_analysis/hits.py +++ b/python/cugraph/cugraph/link_analysis/hits.py @@ -11,11 +11,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -# from cugraph.link_analysis import hits_wrapper - -# from cugraph.utilities import (ensure_cugraph_obj_for_nx, -# df_score_to_dictionary, -# ) +from cugraph.link_analysis import hits_wrapper +from cugraph.utilities import (ensure_cugraph_obj_for_nx, + df_score_to_dictionary, + ) def hits(G, max_iter=100, tol=1.0e-5, nstart=None, normalized=True): @@ -76,10 +75,9 @@ def hits(G, max_iter=100, tol=1.0e-5, nstart=None, normalized=True): >>> hits = cugraph.hits(G, max_iter = 50) """ - """ G, isNx = ensure_cugraph_obj_for_nx(G) - df = hits_wrapper.hits(G, max_iter, tol) # noqa: F821 + df = hits_wrapper.hits(G, max_iter, tol) if G.renumbered: df = G.unrenumber(df, "vertex") @@ -91,5 +89,3 @@ def hits(G, max_iter=100, tol=1.0e-5, nstart=None, normalized=True): df = (d1, d2) return df - """ - raise NotImplementedError("Temporarily disabled. New version in 21.12") diff --git a/python/cugraph/cugraph/link_analysis/hits_wrapper.pyx b/python/cugraph/cugraph/link_analysis/hits_wrapper.pyx index 2a2d33dea0b..d13f34c67d1 100644 --- a/python/cugraph/cugraph/link_analysis/hits_wrapper.pyx +++ b/python/cugraph/cugraph/link_analysis/hits_wrapper.pyx @@ -16,49 +16,93 @@ # cython: embedsignature = True # cython: language_level = 3 -from cugraph.link_analysis.hits cimport hits as c_hits -from cugraph.structure.graph_primtypes cimport * from libc.stdint cimport uintptr_t -from cugraph.structure import graph_primtypes_wrapper +from libcpp.memory cimport unique_ptr + import cudf import numpy as np +from cugraph.structure import graph_primtypes_wrapper +from cugraph.structure.graph_utilities cimport (graph_container_t, + numberTypeEnum, + populate_graph_container, + ) +from cugraph.raft.common.handle cimport handle_t +from cugraph.link_analysis cimport hits as c_hits + def hits(input_graph, max_iter=100, tol=1.0e-5, nstart=None, normalized=True): """ - Call HITS + Call HITS, return a DataFrame containing the hubs and authorities for each + vertex. """ + cdef graph_container_t graph_container + + numberTypeMap = {np.dtype("int32") : numberTypeEnum.int32Type, + np.dtype("int64") : numberTypeEnum.int64Type, + np.dtype("float32") : numberTypeEnum.floatType, + np.dtype("double") : numberTypeEnum.doubleType} + if nstart is not None: raise ValueError('nstart is not currently supported') - if not input_graph.adjlist: - input_graph.view_adj_list() + # Inputs + vertex_t = np.dtype("int32") + edge_t = np.dtype("int32") + weight_t = np.dtype("float32") - [offsets, indices] = graph_primtypes_wrapper.datatype_cast([input_graph.adjlist.offsets, input_graph.adjlist.indices], [np.int32]) + [src, dst] = graph_primtypes_wrapper.datatype_cast( + [input_graph.edgelist.edgelist_df['src'], + input_graph.edgelist.edgelist_df['dst']], + [np.int32]) + weights = None + cdef uintptr_t c_src_vertices = src.__cuda_array_interface__['data'][0] + cdef uintptr_t c_dst_vertices = dst.__cuda_array_interface__['data'][0] + cdef uintptr_t c_edge_weights = NULL num_verts = input_graph.number_of_vertices() num_edges = input_graph.number_of_edges(directed_edges=True) + is_symmetric = not input_graph.is_directed() + cdef unique_ptr[handle_t] handle_ptr + handle_ptr.reset(new handle_t()) + handle_ = handle_ptr.get(); + + populate_graph_container(graph_container, + handle_[0], + c_src_vertices, c_dst_vertices, c_edge_weights, + NULL, + NULL, + 0, + ((numberTypeMap[vertex_t])), + ((numberTypeMap[edge_t])), + ((numberTypeMap[weight_t])), + num_edges, + num_verts, num_edges, + False, + is_symmetric, + False, + False) + + # Outputs df = cudf.DataFrame() - df['vertex'] = cudf.Series(np.zeros(num_verts, dtype=np.int32)) df['hubs'] = cudf.Series(np.zeros(num_verts, dtype=np.float32)) df['authorities'] = cudf.Series(np.zeros(num_verts, dtype=np.float32)) + # The vertex Series is simply the renumbered vertex IDs, which is just 0 to (num_verts-1) + df['vertex'] = cudf.Series(np.arange(num_verts, dtype=np.int32)) - cdef uintptr_t c_identifier = df['vertex'].__cuda_array_interface__['data'][0]; - cdef uintptr_t c_hubs = df['hubs'].__cuda_array_interface__['data'][0]; - cdef uintptr_t c_authorities = df['authorities'].__cuda_array_interface__['data'][0]; - - cdef uintptr_t c_offsets = offsets.__cuda_array_interface__['data'][0] - cdef uintptr_t c_indices = indices.__cuda_array_interface__['data'][0] - cdef uintptr_t c_weights = NULL - - cdef GraphCSRView[int,int,float] graph_float - - graph_float = GraphCSRView[int,int,float](c_offsets, c_indices, c_weights, num_verts, num_edges) + cdef uintptr_t c_hubs_ptr = df['hubs'].__cuda_array_interface__['data'][0]; + cdef uintptr_t c_authorities_ptr = df['authorities'].__cuda_array_interface__['data'][0]; - c_hits[int,int,float](graph_float, max_iter, tol, NULL, - normalized, c_hubs, c_authorities); - graph_float.get_vertex_identifiers(c_identifier) + # Call HITS + c_hits.call_hits[int, float](handle_ptr.get()[0], + graph_container, + c_hubs_ptr, + c_authorities_ptr, + max_iter, + tol, + NULL, + normalized) return df diff --git a/python/cugraph/cugraph/tests/test_hits.py b/python/cugraph/cugraph/tests/test_hits.py index 133485fd174..0d8bff17652 100644 --- a/python/cugraph/cugraph/tests/test_hits.py +++ b/python/cugraph/cugraph/tests/test_hits.py @@ -12,90 +12,100 @@ # limitations under the License. import gc -import time -import pandas as pd import pytest - +import networkx as nx +import pandas as pd import cudf + import cugraph from cugraph.tests import utils -# Temporarily suppress warnings till networkX fixes deprecation warnings -# (Using or importing the ABCs from 'collections' instead of from -# 'collections.abc' is deprecated, and in 3.8 it will stop working) for -# python 3.7. Also, this import networkx needs to be relocated in the -# third-party group once this gets fixed. -import warnings - -with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=DeprecationWarning) - import networkx as nx - - -print("Networkx version : {} ".format(nx.__version__)) - - -def cugraph_call(cu_M, max_iter, tol): - # cugraph hits Call - - t1 = time.time() - G = cugraph.DiGraph() - G.from_cudf_edgelist(cu_M, source="0", destination="1") - df = cugraph.hits(G, max_iter, tol) - df = df.sort_values("vertex").reset_index(drop=True) - t2 = time.time() - t1 - print("Cugraph Time : " + str(t2)) - - return df - - -# The function selects personalization_perc% of accessible vertices in graph M -# and randomly assigns them personalization values -def networkx_call(M, max_iter, tol): - # in NVGRAPH tests we read as CSR and feed as CSC, - # so here we do this explicitly - print("Format conversion ... ") - # Networkx Hits Call - print("Solving... ") - t1 = time.time() - - # Directed NetworkX graph - Gnx = nx.from_pandas_edgelist( - M, source="0", target="1", create_using=nx.DiGraph() - ) - - # same parameters as in NVGRAPH - nx_hits = nx.hits(Gnx, max_iter, tol, normalized=True) - t2 = time.time() - t1 - - print("Networkx Time : " + str(t2)) - - return nx_hits - - -MAX_ITERATIONS = [50] -TOLERANCE = [1.0e-06] - - -@pytest.mark.skip(reason="Waiting on new version") -@pytest.mark.parametrize("graph_file", utils.DATASETS_UNDIRECTED) -@pytest.mark.parametrize("max_iter", MAX_ITERATIONS) -@pytest.mark.parametrize("tol", TOLERANCE) -def test_hits(graph_file, max_iter, tol): +# ============================================================================= +# Pytest Setup / Teardown - called for each test function +# ============================================================================= +def setup_function(): gc.collect() - M = utils.read_csv_for_nx(graph_file) - hubs, authorities = networkx_call(M, max_iter, tol) - cu_M = utils.read_csv_file(graph_file) - cugraph_hits = cugraph_call(cu_M, max_iter, tol) - - pdf = pd.DataFrame.from_dict(hubs, orient="index").sort_index() +# ============================================================================= +# Pytest fixtures +# ============================================================================= +datasets = utils.DATASETS_UNDIRECTED + \ + [utils.RAPIDS_DATASET_ROOT_DIR_PATH/"email-Eu-core.csv"] +fixture_params = utils.genFixtureParamsProduct((datasets, "graph_file"), + ([50], "max_iter"), + ([1.0e-6], "tol"), + ) + + +@pytest.fixture(scope="module", params=fixture_params) +def input_combo(request): + """ + Simply return the current combination of params as a dictionary for use in + tests or other parameterized fixtures. + """ + return dict(zip(("graph_file", "max_iter", "tol"), request.param)) + + +@pytest.fixture(scope="module") +def input_expected_output(input_combo): + """ + This fixture returns a dictionary containing all input params required to + run a HITS algo and the corresponding expected result (based on NetworkX + HITS) which can be used for validation. + """ + # Only run Nx to compute the expected result if it is not already present + # in the dictionary. This allows separate Nx-only tests that may have run + # previously on the same input_combo to save their results for re-use + # elsewhere. + if "nxResults" not in input_combo: + Gnx = utils.generate_nx_graph_from_file(input_combo["graph_file"], + directed=True) + nxResults = nx.hits(Gnx, input_combo["max_iter"], input_combo["tol"], + normalized=True) + input_combo["nxResults"] = nxResults + return input_combo + + +# ============================================================================= +# Tests +# ============================================================================= +def test_nx_hits(benchmark, input_combo): + """ + Simply run NetworkX HITS on the same set of input combinations used for the + cuGraph HITS tests. + This is only in place for generating comparison performance numbers. + """ + Gnx = utils.generate_nx_graph_from_file(input_combo["graph_file"], + directed=True) + nxResults = benchmark( + nx.hits, + Gnx, input_combo["max_iter"], input_combo["tol"], normalized=True + ) + # Save the results back to the input_combo dictionary to prevent redundant + # Nx runs. Other tests using the input_combo fixture will look for them, + # and if not present they will have to re-run the same Nx call. + input_combo["nxResults"] = nxResults + + +def test_hits(benchmark, input_expected_output): + G = utils.generate_cugraph_graph_from_file( + input_expected_output["graph_file"]) + cugraph_hits = benchmark(cugraph.hits, + G, + input_expected_output["max_iter"], + input_expected_output["tol"]) + cugraph_hits = cugraph_hits.sort_values("vertex").reset_index(drop=True) + + (nx_hubs, nx_authorities) = input_expected_output["nxResults"] + + # Update the cugraph HITS results with Nx results for easy comparison using + # cuDF DataFrame methods. + pdf = pd.DataFrame.from_dict(nx_hubs, orient="index").sort_index() cugraph_hits["nx_hubs"] = cudf.Series.from_pandas(pdf[0]) - - pdf = pd.DataFrame.from_dict(authorities, orient="index").sort_index() + pdf = pd.DataFrame.from_dict(nx_authorities, orient="index").sort_index() cugraph_hits["nx_authorities"] = cudf.Series.from_pandas(pdf[0]) hubs_diffs1 = cugraph_hits.query('hubs - nx_hubs > 0.00001') diff --git a/python/cugraph/cugraph/tests/utils.py b/python/cugraph/cugraph/tests/utils.py index 0f1149d9465..a9171a5caec 100755 --- a/python/cugraph/cugraph/tests/utils.py +++ b/python/cugraph/cugraph/tests/utils.py @@ -303,11 +303,10 @@ def generate_nx_graph_from_file(graph_file, directed=True, edgevals=False): return Gnx -def generate_cugraph_graph_from_file( - graph_file, directed=True, edgevals=False -): +def generate_cugraph_graph_from_file(graph_file, directed=True, + edgevals=False): cu_M = read_csv_file(graph_file) - G = cugraph.DiGraph() if directed else cugraph.Graph() + G = cugraph.Graph(directed=directed) if edgevals: G.from_cudf_edgelist(cu_M, source="0", destination="1", edge_attr="2") @@ -434,14 +433,15 @@ def genFixtureParamsProduct(*args): multiple @pytest.mark.parameterize(param_name, param_value_list) decorators. """ - # Enforce that each arg is a list of pytest.param objs and separate params + # Ensure each arg is a list of pytest.param objs, then separate the params # and IDs. paramLists = [] ids = [] paramType = pytest.param().__class__ for (paramList, id) in args: - for param in paramList: - assert isinstance(param, paramType) + for i in range(len(paramList)): + if not isinstance(paramList[i], paramType): + paramList[i] = pytest.param(paramList[i]) paramLists.append(paramList) ids.append(id)