From 7cf65b906d34c672512996f1bb52499c0c0fbe1c Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Thu, 22 Aug 2024 14:09:58 -0700 Subject: [PATCH 1/8] nx-cugraph: add `ZeroGraph` for nx-compatibility (zero-code change) --- python/nx-cugraph/_nx_cugraph/__init__.py | 7 +- python/nx-cugraph/nx_cugraph/__init__.py | 8 + .../nx-cugraph/nx_cugraph/algorithms/core.py | 4 + .../nx_cugraph/algorithms/operators/unary.py | 12 +- .../traversal/breadth_first_search.py | 4 + .../nx-cugraph/nx_cugraph/classes/__init__.py | 3 +- .../nx-cugraph/nx_cugraph/classes/digraph.py | 6 + python/nx-cugraph/nx_cugraph/classes/graph.py | 26 ++- .../nx_cugraph/classes/multidigraph.py | 5 + .../nx_cugraph/classes/multigraph.py | 20 ++ python/nx-cugraph/nx_cugraph/classes/zero.py | 219 ++++++++++++++++++ python/nx-cugraph/nx_cugraph/convert.py | 22 +- .../nx_cugraph/generators/classic.py | 2 + .../nx-cugraph/nx_cugraph/generators/ego.py | 11 +- python/nx-cugraph/nx_cugraph/interface.py | 11 +- python/nx-cugraph/nx_cugraph/relabel.py | 22 +- 16 files changed, 371 insertions(+), 11 deletions(-) create mode 100644 python/nx-cugraph/nx_cugraph/classes/zero.py diff --git a/python/nx-cugraph/_nx_cugraph/__init__.py b/python/nx-cugraph/_nx_cugraph/__init__.py index 41c18c27ecf..16996d4bf42 100644 --- a/python/nx-cugraph/_nx_cugraph/__init__.py +++ b/python/nx-cugraph/_nx_cugraph/__init__.py @@ -22,6 +22,7 @@ $ python _nx_cugraph/__init__.py """ +import os from _nx_cugraph._version import __version__ @@ -293,12 +294,16 @@ def get_info(): for key in info_keys: del d[key] + + d["default_config"] = { + "zero": os.environ.get("NX_CUGRAPH_ZERO", "true").strip().lower() == "true", + } return d def _check_networkx_version(): - import warnings import re + import warnings import networkx as nx diff --git a/python/nx-cugraph/nx_cugraph/__init__.py b/python/nx-cugraph/nx_cugraph/__init__.py index 542256fa781..32de2622612 100644 --- a/python/nx-cugraph/nx_cugraph/__init__.py +++ b/python/nx-cugraph/nx_cugraph/__init__.py @@ -32,7 +32,15 @@ from . import algorithms from .algorithms import * +from .interface import BackendInterface + from _nx_cugraph._version import __git_commit__, __version__ from _nx_cugraph import _check_networkx_version _check_networkx_version() + +BackendInterface.Graph = classes.ZeroGraph +BackendInterface.DiGraph = classes.ZeroDiGraph +BackendInterface.MultiGraph = classes.ZeroMultiGraph +BackendInterface.MultiDiGraph = classes.ZeroMultiDiGraph +del BackendInterface diff --git a/python/nx-cugraph/nx_cugraph/algorithms/core.py b/python/nx-cugraph/nx_cugraph/algorithms/core.py index 8eb9a9946e7..1cf66cf1fa1 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/core.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/core.py @@ -58,7 +58,10 @@ def _(G): @networkx_algorithm(is_incomplete=True, version_added="23.12", _plc="k_truss_subgraph") def k_truss(G, k): if is_nx := isinstance(G, nx.Graph): + zero = isinstance(G, nxcg.ZeroGraph) G = nxcg.from_networkx(G, preserve_all_attrs=True) + else: + zero = False if nxcg.number_of_selfloops(G) > 0: if nx.__version__[:3] <= "3.2": exc_class = nx.NetworkXError @@ -128,6 +131,7 @@ def k_truss(G, k): node_values, node_masks, key_to_id=key_to_id, + zero=zero, ) new_graph.graph.update(G.graph) return new_graph diff --git a/python/nx-cugraph/nx_cugraph/algorithms/operators/unary.py b/python/nx-cugraph/nx_cugraph/algorithms/operators/unary.py index f53b3458949..b09b6f7682a 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/operators/unary.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/operators/unary.py @@ -23,6 +23,7 @@ @networkx_algorithm(version_added="24.02") def complement(G): + zero = isinstance(G, nxcg.ZeroGraph) G = _to_graph(G) N = G._N # Upcast to int64 so indices don't overflow. @@ -43,6 +44,7 @@ def complement(G): src_indices.astype(index_dtype), dst_indices.astype(index_dtype), key_to_id=G.key_to_id, + zero=zero, ) @@ -51,10 +53,16 @@ def reverse(G, copy=True): if not G.is_directed(): raise nx.NetworkXError("Cannot reverse an undirected graph.") if isinstance(G, nx.Graph): - if not copy: + zero = isinstance(G, nxcg.ZeroGraph) + if not copy and not zero: raise RuntimeError( "Using `copy=False` is invalid when using a NetworkX graph " "as input to `nx_cugraph.reverse`" ) G = nxcg.from_networkx(G, preserve_all_attrs=True) - return G.reverse(copy=copy) + else: + zero = False + rv = G.reverse(copy=copy) + if zero: + return rv.to_zero() + return rv diff --git a/python/nx-cugraph/nx_cugraph/algorithms/traversal/breadth_first_search.py b/python/nx-cugraph/nx_cugraph/algorithms/traversal/breadth_first_search.py index 5e4466d7d33..3929dbb2dff 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/traversal/breadth_first_search.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/traversal/breadth_first_search.py @@ -132,6 +132,7 @@ def bfs_tree(G, source, reverse=False, depth_limit=None, sort_neighbors=None): raise NotImplementedError( "sort_neighbors argument in bfs_tree is not currently supported" ) + zero = isinstance(G, nxcg.ZeroGraph) G = _check_G_and_source(G, source) if depth_limit is not None and depth_limit < 1: return nxcg.DiGraph.from_coo( @@ -139,6 +140,7 @@ def bfs_tree(G, source, reverse=False, depth_limit=None, sort_neighbors=None): cp.array([], dtype=index_dtype), cp.array([], dtype=index_dtype), id_to_key=[source], + zero=zero, ) distances, predecessors, node_ids = _bfs( @@ -153,6 +155,7 @@ def bfs_tree(G, source, reverse=False, depth_limit=None, sort_neighbors=None): cp.array([], dtype=index_dtype), cp.array([], dtype=index_dtype), id_to_key=[source], + zero=zero, ) # TODO: create renumbering helper function(s) unique_node_ids = cp.unique(cp.hstack((predecessors, node_ids))) @@ -175,6 +178,7 @@ def bfs_tree(G, source, reverse=False, depth_limit=None, sort_neighbors=None): src_indices, dst_indices, key_to_id=key_to_id, + zero=zero, ) diff --git a/python/nx-cugraph/nx_cugraph/classes/__init__.py b/python/nx-cugraph/nx_cugraph/classes/__init__.py index 19a5357da55..f0ffcd2c850 100644 --- a/python/nx-cugraph/nx_cugraph/classes/__init__.py +++ b/python/nx-cugraph/nx_cugraph/classes/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -10,6 +10,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from .zero import ZeroGraph, ZeroDiGraph, ZeroMultiGraph, ZeroMultiDiGraph from .graph import Graph from .digraph import DiGraph from .multigraph import MultiGraph diff --git a/python/nx-cugraph/nx_cugraph/classes/digraph.py b/python/nx-cugraph/nx_cugraph/classes/digraph.py index e5cfb8f6815..c51109796e7 100644 --- a/python/nx-cugraph/nx_cugraph/classes/digraph.py +++ b/python/nx-cugraph/nx_cugraph/classes/digraph.py @@ -23,6 +23,7 @@ from ..utils import index_dtype from .graph import Graph +from .zero import ZeroDiGraph if TYPE_CHECKING: # pragma: no cover from nx_cugraph.typing import AttrKey @@ -46,6 +47,10 @@ def is_directed(cls) -> bool: def to_networkx_class(cls) -> type[nx.DiGraph]: return nx.DiGraph + @classmethod + def to_zero_class(cls) -> type[ZeroDiGraph]: + return ZeroDiGraph + @networkx_api def size(self, weight: AttrKey | None = None) -> int: if weight is not None: @@ -162,6 +167,7 @@ def to_undirected(self, reciprocal=False, as_view=False): node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + zero=False, ) if as_view: rv.graph = self.graph diff --git a/python/nx-cugraph/nx_cugraph/classes/graph.py b/python/nx-cugraph/nx_cugraph/classes/graph.py index 7425eacb2b4..6b6101a57d3 100644 --- a/python/nx-cugraph/nx_cugraph/classes/graph.py +++ b/python/nx-cugraph/nx_cugraph/classes/graph.py @@ -24,6 +24,7 @@ import nx_cugraph as nxcg from ..utils import index_dtype +from .zero import ZeroGraph if TYPE_CHECKING: # pragma: no cover from collections.abc import Iterable, Iterator @@ -109,6 +110,7 @@ def from_coo( *, key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, + zero: bool | None = None, **attr, ) -> Graph: new_graph = object.__new__(cls) @@ -173,7 +175,8 @@ def from_coo( isolates = nxcg.algorithms.isolate._isolates(new_graph) if len(isolates) > 0: new_graph._node_ids = cp.arange(new_graph._N, dtype=index_dtype) - + if zero or zero is None and nx.config.backends.cugraph.zero: + new_graph = new_graph.to_zero() return new_graph @classmethod @@ -188,6 +191,7 @@ def from_csr( *, key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, + zero: bool | None = None, **attr, ) -> Graph: N = indptr.size - 1 @@ -205,6 +209,7 @@ def from_csr( node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + zero=zero, **attr, ) @@ -220,6 +225,7 @@ def from_csc( *, key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, + zero: bool | None = None, **attr, ) -> Graph: N = indptr.size - 1 @@ -237,6 +243,7 @@ def from_csc( node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + zero=zero, **attr, ) @@ -254,6 +261,7 @@ def from_dcsr( *, key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, + zero: bool | None = None, **attr, ) -> Graph: src_indices = cp.array( @@ -270,6 +278,7 @@ def from_dcsr( node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + zero=zero, **attr, ) @@ -287,6 +296,7 @@ def from_dcsc( *, key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, + zero: bool | None = None, **attr, ) -> Graph: dst_indices = cp.array( @@ -303,13 +313,14 @@ def from_dcsc( node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + zero=zero, **attr, ) def __new__(cls, incoming_graph_data=None, **attr) -> Graph: if incoming_graph_data is None: new_graph = cls.from_coo( - 0, cp.empty(0, index_dtype), cp.empty(0, index_dtype) + 0, cp.empty(0, index_dtype), cp.empty(0, index_dtype), zero=False ) elif incoming_graph_data.__class__ is cls: new_graph = incoming_graph_data.copy() @@ -318,6 +329,7 @@ def __new__(cls, incoming_graph_data=None, **attr) -> Graph: else: raise NotImplementedError new_graph.graph.update(attr) + # XXX: we could return ZeroGraph here, but let's not for now return new_graph ################# @@ -348,6 +360,10 @@ def to_networkx_class(cls) -> type[nx.Graph]: def to_undirected_class(cls) -> type[Graph]: return Graph + @classmethod + def to_zero_class(cls) -> type[ZeroGraph]: + return ZeroGraph + ############## # Properties # ############## @@ -542,6 +558,11 @@ def to_undirected(self, as_view: bool = False) -> Graph: # Does deep copy in networkx return self._copy(as_view, self.to_undirected_class()) + def to_zero(self) -> ZeroGraph: + rv = self.to_zero_class()() + rv._cugraph = self + return rv + # Not implemented... # adj, adjacency, add_edge, add_edges_from, add_node, # add_nodes_from, add_weighted_edges_from, degree, @@ -593,6 +614,7 @@ def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + zero=False, ) if as_view: rv.graph = self.graph diff --git a/python/nx-cugraph/nx_cugraph/classes/multidigraph.py b/python/nx-cugraph/nx_cugraph/classes/multidigraph.py index 2e7a55a9eb1..93deff8b231 100644 --- a/python/nx-cugraph/nx_cugraph/classes/multidigraph.py +++ b/python/nx-cugraph/nx_cugraph/classes/multidigraph.py @@ -18,6 +18,7 @@ from .digraph import DiGraph from .multigraph import MultiGraph +from .zero import ZeroMultiDiGraph __all__ = ["MultiDiGraph"] @@ -34,6 +35,10 @@ def is_directed(cls) -> bool: def to_networkx_class(cls) -> type[nx.MultiDiGraph]: return nx.MultiDiGraph + @classmethod + def to_zero_class(cls) -> type[ZeroMultiDiGraph]: + return ZeroMultiDiGraph + ########################## # NetworkX graph methods # ########################## diff --git a/python/nx-cugraph/nx_cugraph/classes/multigraph.py b/python/nx-cugraph/nx_cugraph/classes/multigraph.py index 23d9faa8734..d738704ccfc 100644 --- a/python/nx-cugraph/nx_cugraph/classes/multigraph.py +++ b/python/nx-cugraph/nx_cugraph/classes/multigraph.py @@ -23,6 +23,7 @@ from ..utils import index_dtype from .graph import Graph +from .zero import ZeroMultiGraph if TYPE_CHECKING: from nx_cugraph.typing import ( @@ -80,6 +81,7 @@ def from_coo( key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, edge_keys: list[EdgeKey] | None = None, + zero: bool | None = None, **attr, ) -> MultiGraph: new_graph = super().from_coo( @@ -92,6 +94,7 @@ def from_coo( node_masks, key_to_id=key_to_id, id_to_key=id_to_key, + zero=False, **attr, ) new_graph.edge_indices = edge_indices @@ -102,6 +105,8 @@ def from_coo( and len(new_graph.edge_keys) != src_indices.size ): raise ValueError + if zero or zero is None and nx.config.backends.cugraph.zero: + new_graph = new_graph.to_zero() return new_graph @classmethod @@ -118,6 +123,7 @@ def from_csr( key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, edge_keys: list[EdgeKey] | None = None, + zero: bool | None = None, **attr, ) -> MultiGraph: N = indptr.size - 1 @@ -137,6 +143,7 @@ def from_csr( key_to_id=key_to_id, id_to_key=id_to_key, edge_keys=edge_keys, + zero=zero, **attr, ) @@ -154,6 +161,7 @@ def from_csc( key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, edge_keys: list[EdgeKey] | None = None, + zero: bool | None = None, **attr, ) -> MultiGraph: N = indptr.size - 1 @@ -173,6 +181,7 @@ def from_csc( key_to_id=key_to_id, id_to_key=id_to_key, edge_keys=edge_keys, + zero=zero, **attr, ) @@ -192,6 +201,7 @@ def from_dcsr( key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, edge_keys: list[EdgeKey] | None = None, + zero: bool | None = None, **attr, ) -> MultiGraph: src_indices = cp.array( @@ -210,6 +220,7 @@ def from_dcsr( key_to_id=key_to_id, id_to_key=id_to_key, edge_keys=edge_keys, + zero=zero, **attr, ) @@ -229,6 +240,7 @@ def from_dcsc( key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, edge_keys: list[EdgeKey] | None = None, + zero: bool | None = None, **attr, ) -> Graph: dst_indices = cp.array( @@ -247,6 +259,7 @@ def from_dcsc( key_to_id=key_to_id, id_to_key=id_to_key, edge_keys=edge_keys, + zero=zero, **attr, ) @@ -261,6 +274,8 @@ def __new__( else: new_graph = super().__new__(cls, incoming_graph_data) new_graph.graph.update(attr) + # if nx.config.backends.cugraph.zero: + # new_graph = new_graph.to_zero() # XXX return new_graph ################# @@ -291,6 +306,10 @@ def to_networkx_class(cls) -> type[nx.MultiGraph]: def to_undirected_class(cls) -> type[MultiGraph]: return MultiGraph + @classmethod + def to_zero_class(cls) -> type[ZeroMultiGraph]: + return ZeroMultiGraph + ########################## # NetworkX graph methods # ########################## @@ -451,6 +470,7 @@ def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): key_to_id=key_to_id, id_to_key=id_to_key, edge_keys=edge_keys, + zero=False, ) if as_view: rv.graph = self.graph diff --git a/python/nx-cugraph/nx_cugraph/classes/zero.py b/python/nx-cugraph/nx_cugraph/classes/zero.py new file mode 100644 index 00000000000..0d20d8464a2 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/classes/zero.py @@ -0,0 +1,219 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import networkx as nx +from networkx.classes.digraph import ( + _CachedPropertyResetterAdjAndSucc, + _CachedPropertyResetterPred, +) +from networkx.classes.graph import ( + _CachedPropertyResetterAdj, + _CachedPropertyResetterNode, +) + +import nx_cugraph as nxcg + +_CACHE_KEY = (True, True) + + +# `collections.UserDict` was the preferred way to subclass dict, but now +# subclassing dict directly is much better supported and should work here. +# TODO: explore if having this class is strictly necessary, but we may choose +# to keep it anyway out of an abundance of caution. +class _ZeroGraphCache(dict): + """Cache that ensures ZeroGraph will reify into a NetworkX graph when cleared.""" + + def __init__(self, zero_graph): + self._zero_graph = zero_graph + + def clear(self): + self._zero_graph._reify_networkx() + super().clear() + + +# TODO: experiment whether we should make `__class__` and other attrs match networkx +class ZeroGraph(nx.Graph): + __networkx_backend__ = "cugraph" + + _nx_attrs = ("_node", "_adj") + + @property + def _node(self): + if (node := self.__dict__["_node"]) is None: + self._reify_networkx() + node = self.__dict__["_node"] + return node + + @_node.setter + def _node(self, val): + _CachedPropertyResetterNode.__set__(None, self, val) + + @property + def _adj(self): + if (adj := self.__dict__["_adj"]) is None: + self._reify_networkx() + adj = self.__dict__["_adj"] + return adj + + @_adj.setter + def _adj(self, val): + _CachedPropertyResetterAdj.__set__(None, self, val) + + @property + def _is_on_gpu(self): + cache = getattr(self, "__networkx_cache__", None) + if not cache: + return False + return _CACHE_KEY in cache.get("backends", {}).get("cugraph", {}) + + @property + def _is_on_cpu(self): + return self.__dict__["_node"] is not None + + @property + def _cugraph(self): + cache = getattr(self, "__networkx_cache__", None) + if cache is None: + cache = {} + cache = cache.setdefault("backends", {}).setdefault("cugraph", {}) + if (Gcg := cache.get(_CACHE_KEY)) is not None: + return Gcg + if self.__dict__["_node"] is None: + raise RuntimeError("XXX") + Gcg = nxcg.from_networkx( + self, preserve_edge_attrs=True, preserve_node_attrs=True + ) + Gcg.graph = self.graph + cache[_CACHE_KEY] = Gcg + return Gcg + + @_cugraph.setter + def _cugraph(self, val): + if (cache := getattr(self, "__networkx_cache__", None)) is None: + # Should we warn? + return + cache = cache.setdefault("backends", {}).setdefault("cugraph", {}) + if val is None: + cache.pop(_CACHE_KEY, None) + else: + self.graph = val.graph + cache[_CACHE_KEY] = val + for key in self._nx_attrs: + self.__dict__[key] = None + + @nx.Graph.name.setter + def name(self, s): + # Don't clear the cache when setting the name, since `.graph` is shared + # XXX + self.graph["name"] = s + + @classmethod + def to_networkx_class(cls): + return nx.Graph + + @classmethod + def to_cugraph_class(cls): + return nxcg.Graph + + def __init__(self, incoming_graph_data=None, **attr): + super().__init__(incoming_graph_data, **attr) + self.__networkx_cache__ = _ZeroGraphCache(self) + + def _reify_networkx(self): + if self.__dict__["_node"] is None: + # After we make this into an nx graph, we rely on the cache being correct + Gcg = self._cugraph + G = nxcg.to_networkx(Gcg) + for key in self._nx_attrs: + self.__dict__[key] = G.__dict__[key] + + +class ZeroDiGraph(nx.DiGraph, ZeroGraph): + _nx_attrs = ("_node", "_adj", "_succ", "_pred") + + name = ZeroGraph.name + _node = ZeroGraph._node + + @property + def _adj(self): + if (adj := self.__dict__["_adj"]) is None: + self._reify_networkx() + adj = self.__dict__["_adj"] + return adj + + @_adj.setter + def _adj(self, val): + _CachedPropertyResetterAdjAndSucc.__set__(None, self, val) + + @property + def _succ(self): + if (succ := self.__dict__["_succ"]) is None: + self._reify_networkx() + succ = self.__dict__["_succ"] + return succ + + @_succ.setter + def _succ(self, val): + _CachedPropertyResetterAdjAndSucc.__set__(None, self, val) + + @property + def _pred(self): + if (pred := self.__dict__["_pred"]) is None: + self._reify_networkx() + pred = self.__dict__["_pred"] + return pred + + @_pred.setter + def _pred(self, val): + _CachedPropertyResetterPred.__set__(None, self, val) + + @classmethod + def to_networkx_class(cls): + return nx.DiGraph + + @classmethod + def to_cugraph_class(cls): + return nxcg.DiGraph + + +class ZeroMultiGraph(nx.MultiGraph, ZeroGraph): + name = ZeroGraph.name + _node = ZeroGraph._node + _adj = ZeroGraph._adj + + @classmethod + def to_networkx_class(cls): + return nx.MultiGraph + + @classmethod + def to_cugraph_class(cls): + return nxcg.MultiGraph + + def __init__(self, incoming_graph_data=None, multigraph_input=None, **attr): + super().__init__(incoming_graph_data, multigraph_input, **attr) + self.__networkx_cache__ = _ZeroGraphCache(self) + + +class ZeroMultiDiGraph(nx.MultiDiGraph, ZeroMultiGraph, ZeroDiGraph): + name = ZeroGraph.name + _node = ZeroGraph._node + _adj = ZeroDiGraph._adj + _succ = ZeroDiGraph._succ + _pred = ZeroDiGraph._pred + + @classmethod + def to_networkx_class(cls): + return nx.MultiDiGraph + + @classmethod + def to_cugraph_class(cls): + return nxcg.MultiDiGraph diff --git a/python/nx-cugraph/nx_cugraph/convert.py b/python/nx-cugraph/nx_cugraph/convert.py index 56d16d837d7..a87a28e883c 100644 --- a/python/nx-cugraph/nx_cugraph/convert.py +++ b/python/nx-cugraph/nx_cugraph/convert.py @@ -24,6 +24,7 @@ import nx_cugraph as nxcg +from .classes.zero import ZeroGraph from .utils import index_dtype, networkx_algorithm from .utils.misc import pairwise @@ -74,6 +75,7 @@ def from_networkx( as_directed: bool = False, name: str | None = None, graph_name: str | None = None, + zero: bool | None = False, ) -> nxcg.Graph: """Convert a networkx graph to nx_cugraph graph; can convert all attributes. @@ -145,6 +147,13 @@ def from_networkx( graph = G else: raise TypeError(f"Expected networkx.Graph; got {type(graph)}") + elif isinstance(graph, ZeroGraph): + if zero or zero is None and nx.config.backends.cugraph.zero: + return graph + if graph._is_on_gpu: + return graph._cugraph + if not graph._is_on_cpu: + raise RuntimeError("TODO") if preserve_all_attrs: preserve_edge_attrs = True @@ -165,7 +174,12 @@ def from_networkx( else: node_attrs = {node_attrs: None} - if graph.__class__ in {nx.Graph, nx.DiGraph, nx.MultiGraph, nx.MultiDiGraph}: + if graph.__class__ in { + nx.Graph, + nx.DiGraph, + nx.MultiGraph, + nx.MultiDiGraph, + } or isinstance(graph, ZeroGraph): # This is a NetworkX private attribute, but is much faster to use adj = graph._adj else: @@ -469,6 +483,7 @@ def func(it, edge_attr=edge_attr, dtype=dtype): node_masks, key_to_id=key_to_id, edge_keys=edge_keys, + zero=zero, ) else: if graph.is_directed() or as_directed: @@ -484,6 +499,7 @@ def func(it, edge_attr=edge_attr, dtype=dtype): node_values, node_masks, key_to_id=key_to_id, + zero=zero, ) if preserve_graph_attrs: rv.graph.update(graph.graph) # deepcopy? @@ -557,6 +573,10 @@ def to_networkx(G: nxcg.Graph, *, sort_edges: bool = False) -> nx.Graph: -------- from_networkx : The opposite; convert networkx graph to nx_cugraph graph """ + if isinstance(G, ZeroGraph): + # ZeroGraphs are already NetworkX graphs :) + # return G # XXX: need to test + G = G._cugraph rv = G.to_networkx_class()() id_to_key = G.id_to_key if sort_edges: diff --git a/python/nx-cugraph/nx_cugraph/generators/classic.py b/python/nx-cugraph/nx_cugraph/generators/classic.py index a548beea34f..b1f83f04ffa 100644 --- a/python/nx-cugraph/nx_cugraph/generators/classic.py +++ b/python/nx-cugraph/nx_cugraph/generators/classic.py @@ -102,6 +102,8 @@ def complete_graph(n, create_using=None): @networkx_algorithm(version_added="23.12") def complete_multipartite_graph(*subset_sizes): if not subset_sizes: + if nx.config.backends.cugraph.zero: + return nxcg.ZeroGraph() return nxcg.Graph() try: subset_sizes = [_ensure_int(size) for size in subset_sizes] diff --git a/python/nx-cugraph/nx_cugraph/generators/ego.py b/python/nx-cugraph/nx_cugraph/generators/ego.py index 66c9c8b95ee..d4e443d1c46 100644 --- a/python/nx-cugraph/nx_cugraph/generators/ego.py +++ b/python/nx-cugraph/nx_cugraph/generators/ego.py @@ -32,7 +32,10 @@ def ego_graph( ): """Weighted ego_graph with negative cycles is not yet supported. `NotImplementedError` will be raised if there are negative `distance` edge weights.""" # noqa: E501 if isinstance(G, nx.Graph): + zero = isinstance(G, nxcg.ZeroGraph) G = nxcg.from_networkx(G, preserve_all_attrs=True) + else: + zero = False if n not in G: if distance is None: raise nx.NodeNotFound(f"Source {n} is not in G") @@ -100,7 +103,10 @@ def ego_graph( node_mask &= node_ids != src_index node_ids = node_ids[node_mask] if node_ids.size == G._N: - return G.copy() + rv = G.copy() + if zero: + return rv.to_zero() + return rv # TODO: create renumbering helper function(s) node_ids.sort() # TODO: is this ever necessary? Keep for safety node_values = {key: val[node_ids] for key, val in G.node_values.items()} @@ -137,6 +143,7 @@ def ego_graph( "node_values": node_values, "node_masks": node_masks, "key_to_id": key_to_id, + "zero": False, } if G.is_multigraph(): if G.edge_keys is not None: @@ -147,6 +154,8 @@ def ego_graph( kwargs["edge_indices"] = G.edge_indices[edge_mask] rv = G.__class__.from_coo(**kwargs) rv.graph.update(G.graph) + if zero: + return rv.to_zero() return rv diff --git a/python/nx-cugraph/nx_cugraph/interface.py b/python/nx-cugraph/nx_cugraph/interface.py index 4007230efa9..0d4b8d6ca07 100644 --- a/python/nx-cugraph/nx_cugraph/interface.py +++ b/python/nx-cugraph/nx_cugraph/interface.py @@ -32,7 +32,16 @@ def convert_from_nx(graph, *args, edge_attrs=None, weight=None, **kwargs): "edge_attrs and weight arguments should not both be given" ) edge_attrs = {weight: 1} - return nxcg.from_networkx(graph, *args, edge_attrs=edge_attrs, **kwargs) + try: + return nxcg.from_networkx( + graph, + *args, + edge_attrs=edge_attrs, + zero=nx.config.backends.cugraph.zero, + **kwargs, + ) + except Exception as exc: + raise NotImplementedError from exc @staticmethod def convert_to_nx(obj, *, name: str | None = None): diff --git a/python/nx-cugraph/nx_cugraph/relabel.py b/python/nx-cugraph/nx_cugraph/relabel.py index 20d1337a99c..75d75b2b624 100644 --- a/python/nx-cugraph/nx_cugraph/relabel.py +++ b/python/nx-cugraph/nx_cugraph/relabel.py @@ -30,12 +30,20 @@ @networkx_algorithm(version_added="24.08") def relabel_nodes(G, mapping, copy=True): if isinstance(G, nx.Graph): - if not copy: + zero = isinstance(G, nxcg.ZeroGraph) + if not copy and not zero: raise RuntimeError( "Using `copy=False` is invalid when using a NetworkX graph " "as input to `nx_cugraph.relabel_nodes`" ) - G = nxcg.from_networkx(G, preserve_all_attrs=True) + try: + G = nxcg.from_networkx(G, preserve_all_attrs=True) + except Exception as exc: + # XXX: this diaper pattern may be generally useful for 'zero'; what's best? + raise NotImplementedError("TODO") from exc + else: + zero = False + it = range(G._N) if G.key_to_id is None else G.id_to_key if callable(mapping): previd_to_key = [mapping(node) for node in it] @@ -225,12 +233,17 @@ def relabel_nodes(G, mapping, copy=True): node_masks=node_masks, id_to_key=newid_to_key, key_to_id=key_to_newid, + zero=False, # XXX TODO: figure out `create_using=` w/ zero graphs **extra_kwargs, ) rv.graph.update(G.graph) if not copy: G._become(rv) + if zero: + return rv.to_zero() # TODO: update original graph return G + if zero: + return rv.to_zero() return rv @@ -241,7 +254,10 @@ def convert_node_labels_to_integers( if ordering not in {"default", "sorted", "increasing degree", "decreasing degree"}: raise nx.NetworkXError(f"Unknown node ordering: {ordering}") if isinstance(G, nx.Graph): + zero = isinstance(G, nxcg.ZeroGraph) G = nxcg.from_networkx(G, preserve_all_attrs=True) + else: + zero = False G = G.copy() if label_attribute is not None: prev_vals = G.id_to_key @@ -279,4 +295,6 @@ def convert_node_labels_to_integers( key_to_id = G.key_to_id G.key_to_id = {i: key_to_id[n] for i, (d, n) in enumerate(pairs, first_label)} G._id_to_key = id_to_key + if zero: + return G.to_zero() return G From e6fb453008043636e83b1866746e1b0dacd7c94a Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Fri, 23 Aug 2024 15:07:47 -0700 Subject: [PATCH 2/8] Better use of caching and handling of versioning; exploring diapers --- .../all_cuda-118_arch-x86_64.yaml | 1 - .../all_cuda-125_arch-x86_64.yaml | 1 - dependencies.yaml | 1 - python/nx-cugraph/_nx_cugraph/__init__.py | 7 ++- python/nx-cugraph/nx_cugraph/__init__.py | 10 ++-- .../algorithms/bipartite/generators.py | 3 +- .../algorithms/community/louvain.py | 4 +- .../nx-cugraph/nx_cugraph/algorithms/core.py | 3 +- .../algorithms/link_analysis/hits_alg.py | 3 +- .../algorithms/shortest_paths/generic.py | 5 +- .../algorithms/shortest_paths/unweighted.py | 5 +- .../traversal/breadth_first_search.py | 3 +- python/nx-cugraph/nx_cugraph/classes/graph.py | 8 ++- .../nx_cugraph/classes/multigraph.py | 10 ++-- python/nx-cugraph/nx_cugraph/classes/zero.py | 30 ++++++++--- python/nx-cugraph/nx_cugraph/convert.py | 51 +++++++++++++++++-- .../nx-cugraph/nx_cugraph/convert_matrix.py | 6 ++- .../nx_cugraph/generators/classic.py | 3 +- python/nx-cugraph/nx_cugraph/interface.py | 33 +++++------- .../nx-cugraph/nx_cugraph/tests/test_bfs.py | 5 +- .../nx_cugraph/tests/test_cluster.py | 5 +- .../nx_cugraph/tests/test_convert.py | 3 -- .../nx_cugraph/tests/test_ego_graph.py | 9 ++-- .../nx_cugraph/tests/test_generators.py | 7 +-- .../nx_cugraph/tests/test_match_api.py | 3 -- .../nx-cugraph/nx_cugraph/utils/decorators.py | 19 +++++-- python/nx-cugraph/pyproject.toml | 2 +- 27 files changed, 157 insertions(+), 83 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 8f9c5ec7a9e..ac5040915e7 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -43,7 +43,6 @@ dependencies: - numpydoc - nvcc_linux-64=11.8 - openmpi -- packaging>=21 - pandas - pre-commit - pydantic diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml index dec67ba4fe4..7adcaf7b468 100644 --- a/conda/environments/all_cuda-125_arch-x86_64.yaml +++ b/conda/environments/all_cuda-125_arch-x86_64.yaml @@ -48,7 +48,6 @@ dependencies: - numpy>=1.23,<2.0a0 - numpydoc - openmpi -- packaging>=21 - pandas - pre-commit - pydantic diff --git a/dependencies.yaml b/dependencies.yaml index 3a7505d3e01..0d2d8c5325d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -649,7 +649,6 @@ dependencies: common: - output_types: [conda, pyproject] packages: - - packaging>=21 # not needed by nx-cugraph tests, but is required for running networkx tests - pytest-mpl cugraph_dgl_dev: diff --git a/python/nx-cugraph/_nx_cugraph/__init__.py b/python/nx-cugraph/_nx_cugraph/__init__.py index 16996d4bf42..4cb603f021d 100644 --- a/python/nx-cugraph/_nx_cugraph/__init__.py +++ b/python/nx-cugraph/_nx_cugraph/__init__.py @@ -301,7 +301,8 @@ def get_info(): return d -def _check_networkx_version(): +def _check_networkx_version() -> tuple[int, int]: + """Check the version of networkx and return ``(major, minor)`` version tuple.""" import re import warnings @@ -326,6 +327,10 @@ def _check_networkx_version(): f"{nx.__version__}. Please upgrade (or fix) your Python environment." ) + nxver_major = int(version_major) + nxver_minor = int(re.match(r"^\d+", version_minor).group()) + return (nxver_major, nxver_minor) + if __name__ == "__main__": from pathlib import Path diff --git a/python/nx-cugraph/nx_cugraph/__init__.py b/python/nx-cugraph/nx_cugraph/__init__.py index 32de2622612..1c218458d6d 100644 --- a/python/nx-cugraph/nx_cugraph/__init__.py +++ b/python/nx-cugraph/nx_cugraph/__init__.py @@ -12,6 +12,11 @@ # limitations under the License. from networkx.exception import * +from _nx_cugraph._version import __git_commit__, __version__ +from _nx_cugraph import _check_networkx_version + +_nxver: tuple[int, int] = _check_networkx_version() + from . import utils from . import classes @@ -34,11 +39,6 @@ from .interface import BackendInterface -from _nx_cugraph._version import __git_commit__, __version__ -from _nx_cugraph import _check_networkx_version - -_check_networkx_version() - BackendInterface.Graph = classes.ZeroGraph BackendInterface.DiGraph = classes.ZeroDiGraph BackendInterface.MultiGraph = classes.ZeroMultiGraph diff --git a/python/nx-cugraph/nx_cugraph/algorithms/bipartite/generators.py b/python/nx-cugraph/nx_cugraph/algorithms/bipartite/generators.py index 60276b7d41b..214970235c6 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/bipartite/generators.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/bipartite/generators.py @@ -16,6 +16,7 @@ import networkx as nx import numpy as np +from nx_cugraph import _nxver from nx_cugraph.generators._utils import _create_using_class, _number_and_nodes from nx_cugraph.utils import index_dtype, networkx_algorithm @@ -48,7 +49,7 @@ def complete_bipartite_graph(n1, n2, create_using=None): nodes.extend(range(n2) if nodes2 is None else nodes2) if len(set(nodes)) != len(nodes): raise nx.NetworkXError("Inputs n1 and n2 must contain distinct nodes") - if nx.__version__[:3] <= "3.3": + if _nxver <= (3, 3): name = f"complete_bipartite_graph({orig_n1}, {orig_n2})" else: name = f"complete_bipartite_graph({n1}, {n2})" diff --git a/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py b/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py index ea1318060e0..482bd32001e 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py @@ -12,9 +12,9 @@ # limitations under the License. import warnings -import networkx as nx import pylibcugraph as plc +from nx_cugraph import _nxver from nx_cugraph.convert import _to_undirected_graph from nx_cugraph.utils import ( _dtype_param, @@ -27,7 +27,7 @@ __all__ = ["louvain_communities"] # max_level argument was added to NetworkX 3.3 -if nx.__version__[:3] <= "3.2": +if _nxver <= (3, 2): _max_level_param = { "max_level : int, optional": ( "Upper limit of the number of macro-iterations (max: 500)." diff --git a/python/nx-cugraph/nx_cugraph/algorithms/core.py b/python/nx-cugraph/nx_cugraph/algorithms/core.py index 1cf66cf1fa1..eff4c6aa4d5 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/core.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/core.py @@ -15,6 +15,7 @@ import pylibcugraph as plc import nx_cugraph as nxcg +from nx_cugraph import _nxver from nx_cugraph.convert import _to_undirected_graph from nx_cugraph.utils import ( _get_int_dtype, @@ -63,7 +64,7 @@ def k_truss(G, k): else: zero = False if nxcg.number_of_selfloops(G) > 0: - if nx.__version__[:3] <= "3.2": + if _nxver <= (3, 2): exc_class = nx.NetworkXError else: exc_class = nx.NetworkXNotImplemented diff --git a/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/hits_alg.py b/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/hits_alg.py index e529b83ab1a..cc59fd5eb64 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/hits_alg.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/link_analysis/hits_alg.py @@ -15,6 +15,7 @@ import numpy as np import pylibcugraph as plc +from nx_cugraph import _nxver from nx_cugraph.convert import _to_graph from nx_cugraph.utils import ( _dtype_param, @@ -53,7 +54,7 @@ def hits( if nstart is not None: nstart = G._dict_to_nodearray(nstart, 0, dtype) if max_iter <= 0: - if nx.__version__[:3] <= "3.2": + if _nxver <= (3, 2): raise ValueError("`maxiter` must be a positive integer.") raise nx.PowerIterationFailedConvergence(max_iter) try: diff --git a/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/generic.py b/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/generic.py index 7d6d77f34a4..ab3c7214303 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/generic.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/generic.py @@ -14,6 +14,7 @@ import numpy as np import nx_cugraph as nxcg +from nx_cugraph import _nxver from nx_cugraph.convert import _to_graph from nx_cugraph.utils import _dtype_param, _get_float_dtype, networkx_algorithm @@ -57,7 +58,7 @@ def shortest_path( paths = nxcg.all_pairs_dijkstra_path(G, weight=weight, dtype=dtype) else: # method == 'bellman-ford': paths = nxcg.all_pairs_bellman_ford_path(G, weight=weight, dtype=dtype) - if nx.__version__[:3] <= "3.4": + if _nxver <= (3, 4): paths = dict(paths) # To target elif method == "unweighted": @@ -129,7 +130,7 @@ def shortest_path_length( # To target elif method == "unweighted": lengths = nxcg.single_target_shortest_path_length(G, target) - if nx.__version__[:3] <= "3.4": + if _nxver <= (3, 4): lengths = dict(lengths) elif method == "dijkstra": lengths = nxcg.single_source_dijkstra_path_length( diff --git a/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/unweighted.py b/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/unweighted.py index 0e98c366e4a..e9c515632ca 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/unweighted.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/shortest_paths/unweighted.py @@ -17,6 +17,7 @@ import numpy as np import pylibcugraph as plc +from nx_cugraph import _nxver from nx_cugraph.convert import _to_graph from nx_cugraph.utils import _groupby, index_dtype, networkx_algorithm @@ -43,7 +44,7 @@ def single_source_shortest_path_length(G, source, cutoff=None): def single_target_shortest_path_length(G, target, cutoff=None): G = _to_graph(G) rv = _bfs(G, target, cutoff, "Target", return_type="length") - if nx.__version__[:3] <= "3.4": + if _nxver <= (3, 4): return iter(rv.items()) return rv @@ -61,7 +62,7 @@ def bidirectional_shortest_path(G, source, target): # TODO PERF: do bidirectional traversal in core G = _to_graph(G) if source not in G or target not in G: - if nx.__version__[:3] <= "3.3": + if _nxver <= (3, 3): raise nx.NodeNotFound( f"Either source {source} or target {target} is not in G" ) diff --git a/python/nx-cugraph/nx_cugraph/algorithms/traversal/breadth_first_search.py b/python/nx-cugraph/nx_cugraph/algorithms/traversal/breadth_first_search.py index 3929dbb2dff..1b1f048faaa 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/traversal/breadth_first_search.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/traversal/breadth_first_search.py @@ -18,6 +18,7 @@ import pylibcugraph as plc import nx_cugraph as nxcg +from nx_cugraph import _nxver from nx_cugraph.convert import _to_graph from nx_cugraph.utils import _groupby, index_dtype, networkx_algorithm @@ -57,7 +58,7 @@ def _bfs(G, source, *, depth_limit=None, reverse=False): return distances[mask], predecessors[mask], node_ids[mask] -if nx.__version__[:3] <= "3.3": +if _nxver <= (3, 3): @networkx_algorithm(is_incomplete=True, version_added="24.02", _plc="bfs") def generic_bfs_edges( diff --git a/python/nx-cugraph/nx_cugraph/classes/graph.py b/python/nx-cugraph/nx_cugraph/classes/graph.py index 6b6101a57d3..74b2a758f31 100644 --- a/python/nx-cugraph/nx_cugraph/classes/graph.py +++ b/python/nx-cugraph/nx_cugraph/classes/graph.py @@ -22,6 +22,7 @@ import pylibcugraph as plc import nx_cugraph as nxcg +from nx_cugraph import _nxver from ..utils import index_dtype from .zero import ZeroGraph @@ -175,7 +176,12 @@ def from_coo( isolates = nxcg.algorithms.isolate._isolates(new_graph) if len(isolates) > 0: new_graph._node_ids = cp.arange(new_graph._N, dtype=index_dtype) - if zero or zero is None and nx.config.backends.cugraph.zero: + if ( + zero + or zero is None + and _nxver >= (3, 3) + and nx.config.backends.cugraph.zero + ): new_graph = new_graph.to_zero() return new_graph diff --git a/python/nx-cugraph/nx_cugraph/classes/multigraph.py b/python/nx-cugraph/nx_cugraph/classes/multigraph.py index d738704ccfc..2c5083256d2 100644 --- a/python/nx-cugraph/nx_cugraph/classes/multigraph.py +++ b/python/nx-cugraph/nx_cugraph/classes/multigraph.py @@ -20,6 +20,7 @@ import numpy as np import nx_cugraph as nxcg +from nx_cugraph import _nxver from ..utils import index_dtype from .graph import Graph @@ -105,7 +106,12 @@ def from_coo( and len(new_graph.edge_keys) != src_indices.size ): raise ValueError - if zero or zero is None and nx.config.backends.cugraph.zero: + if ( + zero + or zero is None + and _nxver >= (3, 3) + and nx.config.backends.cugraph.zero + ): new_graph = new_graph.to_zero() return new_graph @@ -274,8 +280,6 @@ def __new__( else: new_graph = super().__new__(cls, incoming_graph_data) new_graph.graph.update(attr) - # if nx.config.backends.cugraph.zero: - # new_graph = new_graph.to_zero() # XXX return new_graph ################# diff --git a/python/nx-cugraph/nx_cugraph/classes/zero.py b/python/nx-cugraph/nx_cugraph/classes/zero.py index 0d20d8464a2..4c80ea8bcbc 100644 --- a/python/nx-cugraph/nx_cugraph/classes/zero.py +++ b/python/nx-cugraph/nx_cugraph/classes/zero.py @@ -21,8 +21,15 @@ ) import nx_cugraph as nxcg +from nx_cugraph import _nxver -_CACHE_KEY = (True, True) +if _nxver < (3, 4): + _CACHE_KEY = (True, True, True) +else: + _CACHE_KEY = (True, True) + +# Use to indicate when a full conversion to GPU failed so we don't try again. +_CANT_CONVERT_TO_GPU = "_CANT_CONVERT_TO_GPU" # `collections.UserDict` was the preferred way to subclass dict, but now @@ -81,17 +88,24 @@ def _is_on_cpu(self): @property def _cugraph(self): - cache = getattr(self, "__networkx_cache__", None) - if cache is None: - cache = {} - cache = cache.setdefault("backends", {}).setdefault("cugraph", {}) + nx_cache = getattr(self, "__networkx_cache__", None) + if nx_cache is None: + nx_cache = {} + elif _CANT_CONVERT_TO_GPU in nx_cache: + return None + cache = nx_cache.setdefault("backends", {}).setdefault("cugraph", {}) if (Gcg := cache.get(_CACHE_KEY)) is not None: return Gcg if self.__dict__["_node"] is None: raise RuntimeError("XXX") - Gcg = nxcg.from_networkx( - self, preserve_edge_attrs=True, preserve_node_attrs=True - ) + try: + Gcg = nxcg.from_networkx( + self, preserve_edge_attrs=True, preserve_node_attrs=True + ) + except Exception: + # Should we warn that the full graph can't be on GPU? + nx_cache[_CANT_CONVERT_TO_GPU] = True + return None Gcg.graph = self.graph cache[_CACHE_KEY] = Gcg return Gcg diff --git a/python/nx-cugraph/nx_cugraph/convert.py b/python/nx-cugraph/nx_cugraph/convert.py index a87a28e883c..e2b844c00ef 100644 --- a/python/nx-cugraph/nx_cugraph/convert.py +++ b/python/nx-cugraph/nx_cugraph/convert.py @@ -12,6 +12,7 @@ # limitations under the License. from __future__ import annotations +import functools import itertools import operator as op from collections import Counter, defaultdict @@ -23,11 +24,15 @@ import numpy as np import nx_cugraph as nxcg +from nx_cugraph import _nxver from .classes.zero import ZeroGraph from .utils import index_dtype, networkx_algorithm from .utils.misc import pairwise +if _nxver >= (3, 4): + from networkx.utils.backends import _get_cache_key, _get_from_cache, _set_to_cache + if TYPE_CHECKING: # pragma: no cover from nx_cugraph.typing import AttrKey, Dtype, EdgeValue, NodeValue, any_ndarray @@ -61,6 +66,25 @@ def _iterate_values(graph, adj, is_dicts, func): return func(it), False +def _not_implemented_error_diaper(func): + """Catch and convert exceptions to ``NotImplementedError`` + + ``nx.NetworkXError`` are raised without being converted. + """ + + @functools.wraps(func) + def inner(*args, **kwargs): + try: + return func(*args, **kwargs) + except nx.NetworkXError: + raise + except Exception as exc: + raise NotImplementedError from exc + + return inner + + +@_not_implemented_error_diaper def from_networkx( graph: nx.Graph, edge_attrs: AttrKey | dict[AttrKey, EdgeValue | None] | None = None, @@ -148,12 +172,31 @@ def from_networkx( else: raise TypeError(f"Expected networkx.Graph; got {type(graph)}") elif isinstance(graph, ZeroGraph): - if zero or zero is None and nx.config.backends.cugraph.zero: + if ( + zero + or zero is None + and _nxver >= (3, 3) + and nx.config.backends.cugraph.zero + ): return graph if graph._is_on_gpu: return graph._cugraph if not graph._is_on_cpu: raise RuntimeError("TODO") + if _nxver >= (3, 4): + cache_key = _get_cache_key( + edge_attrs=edge_attrs, + node_attrs=node_attrs, + preserve_edge_attrs=preserve_edge_attrs, + preserve_node_attrs=preserve_node_attrs, + preserve_graph_attrs=preserve_graph_attrs, + ) + cache = getattr(graph, "__networkx_cache__", None) + if cache is not None: + cache = cache.setdefault("backends", {}).setdefault("cugraph", {}) + compat_key, rv = _get_from_cache(cache, cache_key) + if rv is not None: + return rv if preserve_all_attrs: preserve_edge_attrs = True @@ -503,6 +546,9 @@ def func(it, edge_attr=edge_attr, dtype=dtype): ) if preserve_graph_attrs: rv.graph.update(graph.graph) # deepcopy? + if _nxver >= (3, 4) and isinstance(graph, ZeroGraph) and cache is not None: + # Make sure this conversion is added to the cache + _set_to_cache(cache, cache_key, rv) return rv @@ -575,8 +621,7 @@ def to_networkx(G: nxcg.Graph, *, sort_edges: bool = False) -> nx.Graph: """ if isinstance(G, ZeroGraph): # ZeroGraphs are already NetworkX graphs :) - # return G # XXX: need to test - G = G._cugraph + return G rv = G.to_networkx_class()() id_to_key = G.id_to_key if sort_edges: diff --git a/python/nx-cugraph/nx_cugraph/convert_matrix.py b/python/nx-cugraph/nx_cugraph/convert_matrix.py index 38139b913cf..e616461b0d0 100644 --- a/python/nx-cugraph/nx_cugraph/convert_matrix.py +++ b/python/nx-cugraph/nx_cugraph/convert_matrix.py @@ -14,6 +14,8 @@ import networkx as nx import numpy as np +from nx_cugraph import _nxver + from .generators._utils import _create_using_class from .utils import _cp_iscopied_asarray, index_dtype, networkx_algorithm @@ -24,7 +26,7 @@ # Value columns with string dtype is not supported -@networkx_algorithm(is_incomplete=True, version_added="23.12") +@networkx_algorithm(is_incomplete=True, version_added="23.12", fallback=True) def from_pandas_edgelist( df, source="source", @@ -138,7 +140,7 @@ def from_pandas_edgelist( and ( # In nx <= 3.3, `edge_key` was ignored if `edge_attr` is None edge_attr is not None - or nx.__version__[:3] > "3.3" + or _nxver > (3, 3) ) ): try: diff --git a/python/nx-cugraph/nx_cugraph/generators/classic.py b/python/nx-cugraph/nx_cugraph/generators/classic.py index b1f83f04ffa..51e37d03285 100644 --- a/python/nx-cugraph/nx_cugraph/generators/classic.py +++ b/python/nx-cugraph/nx_cugraph/generators/classic.py @@ -18,6 +18,7 @@ import numpy as np import nx_cugraph as nxcg +from nx_cugraph import _nxver from ..utils import _get_int_dtype, index_dtype, networkx_algorithm from ._utils import ( @@ -102,7 +103,7 @@ def complete_graph(n, create_using=None): @networkx_algorithm(version_added="23.12") def complete_multipartite_graph(*subset_sizes): if not subset_sizes: - if nx.config.backends.cugraph.zero: + if _nxver >= (3, 3) and nx.config.backends.cugraph.zero: return nxcg.ZeroGraph() return nxcg.Graph() try: diff --git a/python/nx-cugraph/nx_cugraph/interface.py b/python/nx-cugraph/nx_cugraph/interface.py index 0d4b8d6ca07..7110a629fde 100644 --- a/python/nx-cugraph/nx_cugraph/interface.py +++ b/python/nx-cugraph/nx_cugraph/interface.py @@ -18,6 +18,7 @@ import networkx as nx import nx_cugraph as nxcg +from nx_cugraph import _nxver class BackendInterface: @@ -32,20 +33,18 @@ def convert_from_nx(graph, *args, edge_attrs=None, weight=None, **kwargs): "edge_attrs and weight arguments should not both be given" ) edge_attrs = {weight: 1} - try: - return nxcg.from_networkx( - graph, - *args, - edge_attrs=edge_attrs, - zero=nx.config.backends.cugraph.zero, - **kwargs, - ) - except Exception as exc: - raise NotImplementedError from exc + return nxcg.from_networkx( + graph, + *args, + edge_attrs=edge_attrs, + zero=_nxver >= (3, 3) and nx.config.backends.cugraph.zero, + **kwargs, + ) @staticmethod def convert_to_nx(obj, *, name: str | None = None): if isinstance(obj, nxcg.Graph): + # Observe that this does not try to convert ZeroGraph! return nxcg.to_networkx(obj) return obj @@ -184,11 +183,7 @@ def key(testpath): ): no_string_dtype, } - from packaging.version import parse - - nxver = parse(nx.__version__) - - if nxver.major == 3 and nxver.minor <= 2: + if _nxver <= (3, 2): xfail.update( { # NetworkX versions prior to 3.2.1 have tests written to @@ -225,7 +220,7 @@ def key(testpath): } ) - if nxver.major == 3 and nxver.minor <= 1: + if _nxver <= (3, 1): # MAINT: networkx 3.0, 3.1 # NetworkX 3.2 added the ability to "fallback to nx" if backend algorithms # raise NotImplementedError or `can_run` returns False. The tests below @@ -341,7 +336,7 @@ def key(testpath): xfail[key("test_louvain.py:test_threshold")] = ( "Louvain does not support seed parameter" ) - if nxver.major == 3 and nxver.minor >= 2: + if _nxver >= (3, 2): xfail.update( { key( @@ -358,7 +353,7 @@ def key(testpath): ): no_string_dtype, } ) - if nxver.minor == 2: + if _nxver[1] == 2: different_iteration_order = "Different graph data iteration order" xfail.update( { @@ -375,7 +370,7 @@ def key(testpath): ): different_iteration_order, } ) - elif nxver.minor >= 3: + elif _nxver[1] >= 3: xfail.update( { key("test_louvain.py:test_max_level"): louvain_different, diff --git a/python/nx-cugraph/nx_cugraph/tests/test_bfs.py b/python/nx-cugraph/nx_cugraph/tests/test_bfs.py index c2b22e98949..ad2c62c1fb9 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_bfs.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_bfs.py @@ -12,11 +12,10 @@ # limitations under the License. import networkx as nx import pytest -from packaging.version import parse -nxver = parse(nx.__version__) +from nx_cugraph import _nxver -if nxver.major == 3 and nxver.minor < 2: +if _nxver < (3, 2): pytest.skip("Need NetworkX >=3.2 to test clustering", allow_module_level=True) diff --git a/python/nx-cugraph/nx_cugraph/tests/test_cluster.py b/python/nx-cugraph/nx_cugraph/tests/test_cluster.py index ad4770f1ab8..fd8e1b3cf13 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_cluster.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_cluster.py @@ -12,11 +12,10 @@ # limitations under the License. import networkx as nx import pytest -from packaging.version import parse -nxver = parse(nx.__version__) +from nx_cugraph import _nxver -if nxver.major == 3 and nxver.minor < 2: +if _nxver < (3, 2): pytest.skip("Need NetworkX >=3.2 to test clustering", allow_module_level=True) diff --git a/python/nx-cugraph/nx_cugraph/tests/test_convert.py b/python/nx-cugraph/nx_cugraph/tests/test_convert.py index 634b28e961c..3d109af8a74 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_convert.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_convert.py @@ -13,13 +13,10 @@ import cupy as cp import networkx as nx import pytest -from packaging.version import parse import nx_cugraph as nxcg from nx_cugraph import interface -nxver = parse(nx.__version__) - @pytest.mark.parametrize( "graph_class", [nx.Graph, nx.DiGraph, nx.MultiGraph, nx.MultiDiGraph] diff --git a/python/nx-cugraph/nx_cugraph/tests/test_ego_graph.py b/python/nx-cugraph/nx_cugraph/tests/test_ego_graph.py index 5474f9d79e3..c4e5c9cfbb7 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_ego_graph.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_ego_graph.py @@ -12,16 +12,13 @@ # limitations under the License. import networkx as nx import pytest -from packaging.version import parse import nx_cugraph as nxcg +from nx_cugraph import _nxver from .testing_utils import assert_graphs_equal -nxver = parse(nx.__version__) - - -if nxver.major == 3 and nxver.minor < 2: +if _nxver < (3, 2): pytest.skip("Need NetworkX >=3.2 to test ego_graph", allow_module_level=True) @@ -61,7 +58,7 @@ def test_ego_graph_cycle_graph( kwargs["distance"] = "weight" H2nx = nx.ego_graph(Gnx, n, **kwargs) - is_nx32 = nxver.major == 3 and nxver.minor == 2 + is_nx32 = _nxver[:2] == (3, 2) if undirected and Gnx.is_directed() and Gnx.is_multigraph(): if is_nx32: # `should_run` was added in nx 3.3 diff --git a/python/nx-cugraph/nx_cugraph/tests/test_generators.py b/python/nx-cugraph/nx_cugraph/tests/test_generators.py index c751b0fe2b3..2fc97660047 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_generators.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_generators.py @@ -13,16 +13,13 @@ import networkx as nx import numpy as np import pytest -from packaging.version import parse import nx_cugraph as nxcg +from nx_cugraph import _nxver from .testing_utils import assert_graphs_equal -nxver = parse(nx.__version__) - - -if nxver.major == 3 and nxver.minor < 2: +if _nxver < (3, 2): pytest.skip("Need NetworkX >=3.2 to test generators", allow_module_level=True) diff --git a/python/nx-cugraph/nx_cugraph/tests/test_match_api.py b/python/nx-cugraph/nx_cugraph/tests/test_match_api.py index 176b531a6e7..1a61c69b3e7 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_match_api.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_match_api.py @@ -14,13 +14,10 @@ import inspect import networkx as nx -from packaging.version import parse import nx_cugraph as nxcg from nx_cugraph.utils import networkx_algorithm -nxver = parse(nx.__version__) - def test_match_signature_and_names(): """Simple test to ensure our signatures and basic module layout match networkx.""" diff --git a/python/nx-cugraph/nx_cugraph/utils/decorators.py b/python/nx-cugraph/nx_cugraph/utils/decorators.py index 3c5de4f2936..76cb57c5574 100644 --- a/python/nx-cugraph/nx_cugraph/utils/decorators.py +++ b/python/nx-cugraph/nx_cugraph/utils/decorators.py @@ -16,8 +16,10 @@ from textwrap import dedent import networkx as nx +from networkx import NetworkXError from networkx.utils.decorators import nodes_or_number, not_implemented_for +from nx_cugraph import _nxver from nx_cugraph.interface import BackendInterface try: @@ -44,6 +46,7 @@ class networkx_algorithm: version_added: str is_incomplete: bool is_different: bool + _fallback: bool _plc_names: set[str] | None def __new__( @@ -59,6 +62,7 @@ def __new__( version_added: str, # Required is_incomplete: bool = False, # See self.extra_doc for details if True is_different: bool = False, # See self.extra_doc for details if True + fallback: bool = False, # Change non-nx exceptions to NotImplementedError _plc: str | set[str] | None = None, # Hidden from user, may be removed someday ): if func is None: @@ -70,10 +74,11 @@ def __new__( version_added=version_added, is_incomplete=is_incomplete, is_different=is_different, + fallback=fallback, _plc=_plc, ) instance = object.__new__(cls) - if nodes_or_number is not None and nx.__version__[:3] > "3.2": + if nodes_or_number is not None and _nxver > (3, 2): func = nx.utils.decorators.nodes_or_number(nodes_or_number)(func) # update_wrapper sets __wrapped__, which will be used for the signature update_wrapper(instance, func) @@ -100,6 +105,7 @@ def __new__( instance.version_added = version_added instance.is_incomplete = is_incomplete instance.is_different = is_different + instance.fallback = fallback # The docstring on our function is added to the NetworkX docstring. instance.extra_doc = ( dedent(func.__doc__.lstrip("\n").rstrip()) if func.__doc__ else None @@ -113,7 +119,7 @@ def __new__( # Set methods so they are in __dict__ instance._can_run = instance._can_run instance._should_run = instance._should_run - if nodes_or_number is not None and nx.__version__[:3] <= "3.2": + if nodes_or_number is not None and _nxver <= (3, 2): instance = nx.utils.decorators.nodes_or_number(nodes_or_number)(instance) return instance @@ -136,7 +142,14 @@ def _should_run(self, func): self.should_run = func def __call__(self, /, *args, **kwargs): - return self.__wrapped__(*args, **kwargs) + if not self.fallback: + return self.__wrapped__(*args, **kwargs) + try: + return self.__wrapped__(*args, **kwargs) + except NetworkXError: + raise + except Exception as exc: + raise NotImplementedError from exc def __reduce__(self): return _restore_networkx_dispatched, (self.name,) diff --git a/python/nx-cugraph/pyproject.toml b/python/nx-cugraph/pyproject.toml index 847444f9dd1..1c36f5248d9 100644 --- a/python/nx-cugraph/pyproject.toml +++ b/python/nx-cugraph/pyproject.toml @@ -40,7 +40,6 @@ dependencies = [ [project.optional-dependencies] test = [ - "packaging>=21", "pandas", "pytest", "pytest-benchmark", @@ -241,6 +240,7 @@ ignore = [ # Allow assert, print, RNG, and no docstring "nx_cugraph/**/tests/*py" = ["S101", "S311", "T201", "D103", "D100"] "_nx_cugraph/__init__.py" = ["E501"] +"nx_cugraph/__init__.py" = ["E402"] # Allow module level import not at top of file "nx_cugraph/algorithms/**/*py" = ["D205", "D401"] # Allow flexible docstrings for algorithms "nx_cugraph/generators/**/*py" = ["D205", "D401"] # Allow flexible docstrings for generators "nx_cugraph/interface.py" = ["D401"] # Flexible docstrings From 0b80ccfef123b807d953da2050de5baa80eb3595 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sat, 24 Aug 2024 13:28:19 -0700 Subject: [PATCH 3/8] Some clean up; try to make tests in CI pass --- .../algorithms/community/louvain.py | 2 +- python/nx-cugraph/nx_cugraph/classes/graph.py | 2 +- python/nx-cugraph/nx_cugraph/classes/zero.py | 17 +- python/nx-cugraph/nx_cugraph/convert.py | 23 +- .../nx-cugraph/nx_cugraph/convert_matrix.py | 2 +- python/nx-cugraph/nx_cugraph/interface.py | 219 +++++++++++------- python/nx-cugraph/nx_cugraph/relabel.py | 6 +- python/nx-cugraph/run_nx_tests.sh | 4 + 8 files changed, 167 insertions(+), 108 deletions(-) diff --git a/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py b/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py index 482bd32001e..52c512c454d 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/community/louvain.py @@ -81,7 +81,7 @@ def _louvain_communities( node_ids, clusters, modularity = plc.louvain( resource_handle=plc.ResourceHandle(), graph=G._get_plc_graph(weight, 1, dtype), - max_level=max_level, # TODO: add this parameter to NetworkX + max_level=max_level, threshold=threshold, resolution=resolution, do_expensive_check=False, diff --git a/python/nx-cugraph/nx_cugraph/classes/graph.py b/python/nx-cugraph/nx_cugraph/classes/graph.py index 74b2a758f31..2abf0cd6502 100644 --- a/python/nx-cugraph/nx_cugraph/classes/graph.py +++ b/python/nx-cugraph/nx_cugraph/classes/graph.py @@ -335,7 +335,7 @@ def __new__(cls, incoming_graph_data=None, **attr) -> Graph: else: raise NotImplementedError new_graph.graph.update(attr) - # XXX: we could return ZeroGraph here, but let's not for now + # We could return ZeroGraph here (if configured), but let's not for now return new_graph ################# diff --git a/python/nx-cugraph/nx_cugraph/classes/zero.py b/python/nx-cugraph/nx_cugraph/classes/zero.py index 4c80ea8bcbc..d3968766816 100644 --- a/python/nx-cugraph/nx_cugraph/classes/zero.py +++ b/python/nx-cugraph/nx_cugraph/classes/zero.py @@ -34,8 +34,7 @@ # `collections.UserDict` was the preferred way to subclass dict, but now # subclassing dict directly is much better supported and should work here. -# TODO: explore if having this class is strictly necessary, but we may choose -# to keep it anyway out of an abundance of caution. +# This class should only be necessary if the user clears the cache manually. class _ZeroGraphCache(dict): """Cache that ensures ZeroGraph will reify into a NetworkX graph when cleared.""" @@ -97,7 +96,12 @@ def _cugraph(self): if (Gcg := cache.get(_CACHE_KEY)) is not None: return Gcg if self.__dict__["_node"] is None: - raise RuntimeError("XXX") + raise RuntimeError( + f"{type(self).__name__} cannot be converted to the GPU, because it is " + "not on the CPU! This is not supposed to be possible. If you believe " + "you have found a bug, please report a minimum reproducible example to " + "https://github.com/rapidsai/cugraph/issues/new/choose" + ) try: Gcg = nxcg.from_networkx( self, preserve_edge_attrs=True, preserve_node_attrs=True @@ -115,6 +119,8 @@ def _cugraph(self, val): if (cache := getattr(self, "__networkx_cache__", None)) is None: # Should we warn? return + # TODO: pay close attention to when we should clear the cache, since + # this may or may not be a mutation. cache = cache.setdefault("backends", {}).setdefault("cugraph", {}) if val is None: cache.pop(_CACHE_KEY, None) @@ -126,8 +132,9 @@ def _cugraph(self, val): @nx.Graph.name.setter def name(self, s): - # Don't clear the cache when setting the name, since `.graph` is shared - # XXX + # Don't clear the cache when setting the name, since `.graph` is shared. + # There is a very small risk here for the cache to become (slightly) + # insconsistent if graphs from other backends are cached. self.graph["name"] = s @classmethod diff --git a/python/nx-cugraph/nx_cugraph/convert.py b/python/nx-cugraph/nx_cugraph/convert.py index e2b844c00ef..1abe112ea47 100644 --- a/python/nx-cugraph/nx_cugraph/convert.py +++ b/python/nx-cugraph/nx_cugraph/convert.py @@ -66,10 +66,12 @@ def _iterate_values(graph, adj, is_dicts, func): return func(it), False -def _not_implemented_error_diaper(func): - """Catch and convert exceptions to ``NotImplementedError`` +# Consider adding this to `utils` if it is useful elsewhere +def _fallback_decorator(func): + """Catch and convert exceptions to ``NotImplementedError``; use as a decorator. - ``nx.NetworkXError`` are raised without being converted. + ``nx.NetworkXError`` are raised without being converted. This allows + falling back to other backends if, for example, conversion to GPU failed. """ @functools.wraps(func) @@ -84,7 +86,7 @@ def inner(*args, **kwargs): return inner -@_not_implemented_error_diaper +@_fallback_decorator def from_networkx( graph: nx.Graph, edge_attrs: AttrKey | dict[AttrKey, EdgeValue | None] | None = None, @@ -182,7 +184,12 @@ def from_networkx( if graph._is_on_gpu: return graph._cugraph if not graph._is_on_cpu: - raise RuntimeError("TODO") + raise RuntimeError( + f"{type(graph).__name__} cannot be converted to the GPU, because it is " + "not on the CPU! This is not supposed to be possible. If you believe " + "you have found a bug, please report a minimum reproducible example to " + "https://github.com/rapidsai/cugraph/issues/new/choose" + ) if _nxver >= (3, 4): cache_key = _get_cache_key( edge_attrs=edge_attrs, @@ -547,7 +554,9 @@ def func(it, edge_attr=edge_attr, dtype=dtype): if preserve_graph_attrs: rv.graph.update(graph.graph) # deepcopy? if _nxver >= (3, 4) and isinstance(graph, ZeroGraph) and cache is not None: - # Make sure this conversion is added to the cache + # Make sure this conversion is added to the cache, and make all of + # our graphs share the same `.graph` attribute for consistency. + rv.graph = graph.graph _set_to_cache(cache, cache_key, rv) return rv @@ -753,7 +762,7 @@ def _to_undirected_graph( raise TypeError -@networkx_algorithm(version_added="24.08") +@networkx_algorithm(version_added="24.08", fallback=True) def from_dict_of_lists(d, create_using=None): from .generators._utils import _create_using_class diff --git a/python/nx-cugraph/nx_cugraph/convert_matrix.py b/python/nx-cugraph/nx_cugraph/convert_matrix.py index e616461b0d0..54975902861 100644 --- a/python/nx-cugraph/nx_cugraph/convert_matrix.py +++ b/python/nx-cugraph/nx_cugraph/convert_matrix.py @@ -163,7 +163,7 @@ def from_pandas_edgelist( return G -@networkx_algorithm(version_added="23.12") +@networkx_algorithm(version_added="23.12", fallback=True) def from_scipy_sparse_array( A, parallel_edges=False, create_using=None, edge_attribute="weight" ): diff --git a/python/nx-cugraph/nx_cugraph/interface.py b/python/nx-cugraph/nx_cugraph/interface.py index 7110a629fde..8910e463904 100644 --- a/python/nx-cugraph/nx_cugraph/interface.py +++ b/python/nx-cugraph/nx_cugraph/interface.py @@ -70,19 +70,30 @@ def key(testpath): return (testname, frozenset({classname, filename})) return (testname, frozenset({filename})) + zero = _nxver >= (3, 3) and nx.config.backends.cugraph.zero + fallback = zero or nx.utils.backends._dispatchable._fallback_to_nx + # Reasons for xfailing + # For nx version <= 3.1 no_weights = "weighted implementation not currently supported" no_multigraph = "multigraphs not currently supported" + # For nx version <= 3.2 + nx_cugraph_in_test_setup = ( + "nx-cugraph Graph is incompatible in test setup in nx versions < 3.3" + ) + # For all versions louvain_different = "Louvain may be different due to RNG" - no_string_dtype = "string edge values not currently supported" sssp_path_different = "sssp may choose a different valid path" + tuple_elements_preferred = "elements are tuples instead of lists" + no_mixed_dtypes_for_nodes = ( + # This one is tricky b/c we don't raise; all dtypes are treated as str + "mixed dtypes (str, int, float) for single node property not supported" + ) + # These shouldn't fail if using ZeroGraph or falling back to networkx + no_string_dtype = "string edge values not currently supported" no_object_dtype_for_edges = ( "Edges don't support object dtype (lists, strings, etc.)" ) - tuple_elements_preferred = "elements are tuples instead of lists" - nx_cugraph_in_test_setup = ( - "nx-cugraph Graph is incompatible in test setup in nx versions < 3.3" - ) xfail = { # This is removed while strongly_connected_components() is not @@ -106,38 +117,6 @@ def key(testpath): "test_cycles.py:TestMinimumCycleBasis." "test_gh6787_and_edge_attribute_names" ): sssp_path_different, - key( - "test_graph_hashing.py:test_isomorphic_edge_attr" - ): no_object_dtype_for_edges, - key( - "test_graph_hashing.py:test_isomorphic_edge_attr_and_node_attr" - ): no_object_dtype_for_edges, - key( - "test_graph_hashing.py:test_isomorphic_edge_attr_subgraph_hash" - ): no_object_dtype_for_edges, - key( - "test_graph_hashing.py:" - "test_isomorphic_edge_attr_and_node_attr_subgraph_hash" - ): no_object_dtype_for_edges, - key( - "test_summarization.py:TestSNAPNoEdgeTypes.test_summary_graph" - ): no_object_dtype_for_edges, - key( - "test_summarization.py:TestSNAPUndirected.test_summary_graph" - ): no_object_dtype_for_edges, - key( - "test_summarization.py:TestSNAPDirected.test_summary_graph" - ): no_object_dtype_for_edges, - key("test_gexf.py:TestGEXF.test_relabel"): no_object_dtype_for_edges, - key( - "test_gml.py:TestGraph.test_parse_gml_cytoscape_bug" - ): no_object_dtype_for_edges, - key("test_gml.py:TestGraph.test_parse_gml"): no_object_dtype_for_edges, - key("test_gml.py:TestGraph.test_read_gml"): no_object_dtype_for_edges, - key("test_gml.py:TestGraph.test_data_types"): no_object_dtype_for_edges, - key( - "test_gml.py:TestPropertyLists.test_reading_graph_with_list_property" - ): no_object_dtype_for_edges, key( "test_relabel.py:" "test_relabel_preserve_node_order_partial_mapping_with_copy_false" @@ -146,42 +125,105 @@ def key(testpath): "test_gml.py:" "TestPropertyLists.test_reading_graph_with_single_element_list_property" ): tuple_elements_preferred, - key( - "test_relabel.py:" - "TestRelabel.test_relabel_multidigraph_inout_merge_nodes" - ): no_string_dtype, - key( - "test_relabel.py:TestRelabel.test_relabel_multigraph_merge_inplace" - ): no_string_dtype, - key( - "test_relabel.py:TestRelabel.test_relabel_multidigraph_merge_inplace" - ): no_string_dtype, - key( - "test_relabel.py:TestRelabel.test_relabel_multidigraph_inout_copy" - ): no_string_dtype, - key( - "test_relabel.py:TestRelabel.test_relabel_multigraph_merge_copy" - ): no_string_dtype, - key( - "test_relabel.py:TestRelabel.test_relabel_multidigraph_merge_copy" - ): no_string_dtype, - key( - "test_relabel.py:TestRelabel.test_relabel_multigraph_nonnumeric_key" - ): no_string_dtype, - key("test_contraction.py:test_multigraph_path"): no_object_dtype_for_edges, - key( - "test_contraction.py:test_directed_multigraph_path" - ): no_object_dtype_for_edges, - key( - "test_contraction.py:test_multigraph_blockmodel" - ): no_object_dtype_for_edges, - key( - "test_summarization.py:TestSNAPUndirectedMulti.test_summary_graph" - ): no_string_dtype, - key( - "test_summarization.py:TestSNAPDirectedMulti.test_summary_graph" - ): no_string_dtype, } + if not fallback: + xfail.update( + { + key( + "test_graph_hashing.py:test_isomorphic_edge_attr" + ): no_object_dtype_for_edges, + key( + "test_graph_hashing.py:test_isomorphic_edge_attr_and_node_attr" + ): no_object_dtype_for_edges, + key( + "test_graph_hashing.py:test_isomorphic_edge_attr_subgraph_hash" + ): no_object_dtype_for_edges, + key( + "test_graph_hashing.py:" + "test_isomorphic_edge_attr_and_node_attr_subgraph_hash" + ): no_object_dtype_for_edges, + key( + "test_summarization.py:TestSNAPNoEdgeTypes.test_summary_graph" + ): no_object_dtype_for_edges, + key( + "test_summarization.py:TestSNAPUndirected.test_summary_graph" + ): no_object_dtype_for_edges, + key( + "test_summarization.py:TestSNAPDirected.test_summary_graph" + ): no_object_dtype_for_edges, + key( + "test_gexf.py:TestGEXF.test_relabel" + ): no_object_dtype_for_edges, + key( + "test_gml.py:TestGraph.test_parse_gml_cytoscape_bug" + ): no_object_dtype_for_edges, + key( + "test_gml.py:TestGraph.test_parse_gml" + ): no_object_dtype_for_edges, + key( + "test_gml.py:TestGraph.test_read_gml" + ): no_object_dtype_for_edges, + key( + "test_gml.py:TestGraph.test_data_types" + ): no_object_dtype_for_edges, + key( + "test_gml.py:" + "TestPropertyLists.test_reading_graph_with_list_property" + ): no_object_dtype_for_edges, + key( + "test_relabel.py:" + "TestRelabel.test_relabel_multidigraph_inout_merge_nodes" + ): no_string_dtype, + key( + "test_relabel.py:" + "TestRelabel.test_relabel_multigraph_merge_inplace" + ): no_string_dtype, + key( + "test_relabel.py:" + "TestRelabel.test_relabel_multidigraph_merge_inplace" + ): no_string_dtype, + key( + "test_relabel.py:" + "TestRelabel.test_relabel_multidigraph_inout_copy" + ): no_string_dtype, + key( + "test_relabel.py:TestRelabel.test_relabel_multigraph_merge_copy" + ): no_string_dtype, + key( + "test_relabel.py:" + "TestRelabel.test_relabel_multidigraph_merge_copy" + ): no_string_dtype, + key( + "test_relabel.py:" + "TestRelabel.test_relabel_multigraph_nonnumeric_key" + ): no_string_dtype, + key( + "test_contraction.py:test_multigraph_path" + ): no_object_dtype_for_edges, + key( + "test_contraction.py:test_directed_multigraph_path" + ): no_object_dtype_for_edges, + key( + "test_contraction.py:test_multigraph_blockmodel" + ): no_object_dtype_for_edges, + key( + "test_summarization.py:" + "TestSNAPUndirectedMulti.test_summary_graph" + ): no_string_dtype, + key( + "test_summarization.py:TestSNAPDirectedMulti.test_summary_graph" + ): no_string_dtype, + } + ) + else: + xfail.update( + { + key( + "test_gml.py:" + "TestPropertyLists.test_reading_graph_with_list_property" + ): no_mixed_dtypes_for_nodes, + } + ) if _nxver <= (3, 2): xfail.update( @@ -337,22 +379,23 @@ def key(testpath): "Louvain does not support seed parameter" ) if _nxver >= (3, 2): - xfail.update( - { - key( - "test_convert_pandas.py:TestConvertPandas." - "test_from_edgelist_multi_attr_incl_target" - ): no_string_dtype, - key( - "test_convert_pandas.py:TestConvertPandas." - "test_from_edgelist_multidigraph_and_edge_attr" - ): no_string_dtype, - key( - "test_convert_pandas.py:TestConvertPandas." - "test_from_edgelist_int_attr_name" - ): no_string_dtype, - } - ) + if not fallback: + xfail.update( + { + key( + "test_convert_pandas.py:TestConvertPandas." + "test_from_edgelist_multi_attr_incl_target" + ): no_string_dtype, + key( + "test_convert_pandas.py:TestConvertPandas." + "test_from_edgelist_multidigraph_and_edge_attr" + ): no_string_dtype, + key( + "test_convert_pandas.py:TestConvertPandas." + "test_from_edgelist_int_attr_name" + ): no_string_dtype, + } + ) if _nxver[1] == 2: different_iteration_order = "Different graph data iteration order" xfail.update( diff --git a/python/nx-cugraph/nx_cugraph/relabel.py b/python/nx-cugraph/nx_cugraph/relabel.py index 75d75b2b624..fe18183dd45 100644 --- a/python/nx-cugraph/nx_cugraph/relabel.py +++ b/python/nx-cugraph/nx_cugraph/relabel.py @@ -36,11 +36,7 @@ def relabel_nodes(G, mapping, copy=True): "Using `copy=False` is invalid when using a NetworkX graph " "as input to `nx_cugraph.relabel_nodes`" ) - try: - G = nxcg.from_networkx(G, preserve_all_attrs=True) - except Exception as exc: - # XXX: this diaper pattern may be generally useful for 'zero'; what's best? - raise NotImplementedError("TODO") from exc + G = nxcg.from_networkx(G, preserve_all_attrs=True) else: zero = False diff --git a/python/nx-cugraph/run_nx_tests.sh b/python/nx-cugraph/run_nx_tests.sh index bceec53b7d5..b67343c7d3c 100755 --- a/python/nx-cugraph/run_nx_tests.sh +++ b/python/nx-cugraph/run_nx_tests.sh @@ -18,6 +18,10 @@ # testing takes longer. Without it, tests will xfail when encountering a # function that we don't implement. # +# NX_CUGRAPH_ZERO, {"True", "False"}, default is "True" +# Whether to use `nxcg.ZeroGraph` as the nx_cugraph backend graph. +# A ZeroGraph should be a compatible NetworkX graph, so fewer tests should fail. +# # Coverage of `nx_cugraph.algorithms` is reported and is a good sanity check # that algorithms run. From 24a9788b2668773c6e8c0d1e205945597d1becd9 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Sun, 25 Aug 2024 13:41:33 -0700 Subject: [PATCH 4/8] Try to get CI to pass --- ci/test_python.sh | 2 +- python/nx-cugraph/nx_cugraph/convert.py | 4 +-- .../nx-cugraph/nx_cugraph/utils/decorators.py | 4 ++- python/nx-cugraph/nx_cugraph/utils/misc.py | 34 +++++++++++++++++++ 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/ci/test_python.sh b/ci/test_python.sh index e8c8272e8d6..83138e96035 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -108,7 +108,7 @@ echo "nx-cugraph coverage from networkx tests: $_coverage" echo $_coverage | awk '{ if ($NF == "0.0%") exit 1 }' # Ensure all algorithms were called by comparing covered lines to function lines. # Run our tests again (they're fast enough) to add their coverage, then create coverage.json -pytest \ +NX_CUGRAPH_ZERO=False pytest \ --pyargs nx_cugraph \ --config-file=../pyproject.toml \ --cov-config=../pyproject.toml \ diff --git a/python/nx-cugraph/nx_cugraph/convert.py b/python/nx-cugraph/nx_cugraph/convert.py index 1abe112ea47..c122bca2c07 100644 --- a/python/nx-cugraph/nx_cugraph/convert.py +++ b/python/nx-cugraph/nx_cugraph/convert.py @@ -28,7 +28,7 @@ from .classes.zero import ZeroGraph from .utils import index_dtype, networkx_algorithm -from .utils.misc import pairwise +from .utils.misc import _And_NotImplementedError, pairwise if _nxver >= (3, 4): from networkx.utils.backends import _get_cache_key, _get_from_cache, _set_to_cache @@ -81,7 +81,7 @@ def inner(*args, **kwargs): except nx.NetworkXError: raise except Exception as exc: - raise NotImplementedError from exc + raise _And_NotImplementedError(exc) from exc return inner diff --git a/python/nx-cugraph/nx_cugraph/utils/decorators.py b/python/nx-cugraph/nx_cugraph/utils/decorators.py index 76cb57c5574..16486996ba0 100644 --- a/python/nx-cugraph/nx_cugraph/utils/decorators.py +++ b/python/nx-cugraph/nx_cugraph/utils/decorators.py @@ -22,6 +22,8 @@ from nx_cugraph import _nxver from nx_cugraph.interface import BackendInterface +from .misc import _And_NotImplementedError + try: from networkx.utils.backends import _registered_algorithms except ModuleNotFoundError: @@ -149,7 +151,7 @@ def __call__(self, /, *args, **kwargs): except NetworkXError: raise except Exception as exc: - raise NotImplementedError from exc + raise _And_NotImplementedError(exc) from exc def __reduce__(self): return _restore_networkx_dispatched, (self.name,) diff --git a/python/nx-cugraph/nx_cugraph/utils/misc.py b/python/nx-cugraph/nx_cugraph/utils/misc.py index 8526524f1de..4831fe93894 100644 --- a/python/nx-cugraph/nx_cugraph/utils/misc.py +++ b/python/nx-cugraph/nx_cugraph/utils/misc.py @@ -238,3 +238,37 @@ def _cp_iscopied_asarray(a, *args, orig_object=None, **kwargs): ): return False, arr return True, arr + + +class _And_NotImplementedError(NotImplementedError): + """Additionally make an exception a ``NotImplementedError``. + + For example: + + >>> try: + ... raise _And_NotImplementedError(KeyError("missing")) + ... except KeyError: + ... pass + + or + + >>> try: + ... raise _And_NotImplementedError(KeyError("missing")) + ... except NotImplementedError: + ... pass + + """ + + def __new__(cls, exc): + exc_type = type(exc) + if issubclass(exc_type, NotImplementedError): + new_type = exc_type + else: + new_type = type( + f"{exc_type.__name__}{cls.__name__}", + (exc_type, NotImplementedError), + {}, + ) + instance = NotImplementedError.__new__(new_type) + instance.__init__(*exc.args) + return instance From 445a4da221476e5d01bb2a4aa764ce2a8fb02d62 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 27 Aug 2024 18:21:42 +0200 Subject: [PATCH 5/8] Try to get CI to pass by setting NX_CUGRAPH_ZERO=False --- ci/run_nx_cugraph_pytests.sh | 2 +- ci/test_wheel.sh | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/run_nx_cugraph_pytests.sh b/ci/run_nx_cugraph_pytests.sh index b0caffd0a0f..52c6782c4ad 100755 --- a/ci/run_nx_cugraph_pytests.sh +++ b/ci/run_nx_cugraph_pytests.sh @@ -6,4 +6,4 @@ set -euo pipefail # Support invoking run_nx_cugraph_pytests.sh outside the script directory cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/nx-cugraph/nx_cugraph -pytest --capture=no --cache-clear --benchmark-disable "$@" tests +NX_CUGRAPH_ZERO=False pytest --capture=no --cache-clear --benchmark-disable "$@" tests diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 158704e08d1..f93052a410b 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -37,6 +37,7 @@ else DASK_DISTRIBUTED__SCHEDULER__WORKER_TTL="1000s" \ DASK_DISTRIBUTED__COMM__TIMEOUTS__CONNECT="1000s" \ DASK_CUDA_WAIT_WORKERS_MIN_TIMEOUT="1000s" \ + NX_CUGRAPH_ZERO=False \ python -m pytest \ -v \ --import-mode=append \ From b6f33089160c7b1406bb7dc11a01ef2f63a42f29 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Tue, 27 Aug 2024 18:28:18 +0200 Subject: [PATCH 6/8] Update linting (dropped Python 3.9) --- python/nx-cugraph/lint.yaml | 2 +- python/nx-cugraph/pyproject.toml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/python/nx-cugraph/lint.yaml b/python/nx-cugraph/lint.yaml index ce46360e234..b2184a185c4 100644 --- a/python/nx-cugraph/lint.yaml +++ b/python/nx-cugraph/lint.yaml @@ -43,7 +43,7 @@ repos: rev: v3.16.0 hooks: - id: pyupgrade - args: [--py39-plus] + args: [--py310-plus] - repo: https://github.com/psf/black rev: 24.4.2 hooks: diff --git a/python/nx-cugraph/pyproject.toml b/python/nx-cugraph/pyproject.toml index 4f0652964a4..e356735d25f 100644 --- a/python/nx-cugraph/pyproject.toml +++ b/python/nx-cugraph/pyproject.toml @@ -168,6 +168,7 @@ external = [ ] ignore = [ # Would be nice to fix these + "B905", # `zip()` without an explicit `strict=` parameter (Note: possible since py39 was dropped; we should do this!) "D100", # Missing docstring in public module "D101", # Missing docstring in public class "D102", # Missing docstring in public method @@ -213,6 +214,7 @@ ignore = [ "SIM105", # Use contextlib.suppress(...) instead of try-except-pass (Note: try-except-pass is much faster) "SIM108", # Use ternary operator ... instead of if-else-block (Note: if-else better for coverage and sometimes clearer) "TRY003", # Avoid specifying long messages outside the exception class (Note: why?) + "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)` (Note: tuple is faster for now) # Ignored categories "C90", # mccabe (Too strict, but maybe we should make things less complex) From d615fedacacc46e3dd891514af8e8ad561bb5ffb Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Mon, 9 Sep 2024 05:09:13 -0700 Subject: [PATCH 7/8] Rename `Graph`->`CudaGraph` and `ZeroGraph`->`Graph`; many improvements In addition to renaming and moving graph classes around, this smooths out *many* rough edges for zero-code change use cases. There is more to be done, but this is excellent progress and should be good enough to use. Now to see about getting CI to pass! --- ci/run_nx_cugraph_pytests.sh | 3 +- ci/test_python.sh | 2 +- ci/test_wheel.sh | 2 +- python/nx-cugraph/_nx_cugraph/__init__.py | 5 +- python/nx-cugraph/lint.yaml | 16 +- python/nx-cugraph/nx_cugraph/__init__.py | 8 +- .../nx-cugraph/nx_cugraph/algorithms/core.py | 6 +- .../nx_cugraph/algorithms/operators/unary.py | 14 +- .../traversal/breadth_first_search.py | 14 +- .../nx-cugraph/nx_cugraph/classes/__init__.py | 9 +- .../nx-cugraph/nx_cugraph/classes/digraph.py | 91 +++- python/nx-cugraph/nx_cugraph/classes/graph.py | 411 ++++++++++++++---- .../nx_cugraph/classes/multidigraph.py | 36 +- .../nx_cugraph/classes/multigraph.py | 176 +++++--- python/nx-cugraph/nx_cugraph/classes/zero.py | 240 ---------- python/nx-cugraph/nx_cugraph/convert.py | 82 ++-- .../nx_cugraph/generators/_utils.py | 16 +- .../nx_cugraph/generators/classic.py | 8 +- .../nx_cugraph/generators/community.py | 7 +- .../nx-cugraph/nx_cugraph/generators/ego.py | 14 +- .../nx-cugraph/nx_cugraph/generators/small.py | 10 +- .../nx_cugraph/generators/social.py | 29 +- python/nx-cugraph/nx_cugraph/interface.py | 15 +- python/nx-cugraph/nx_cugraph/relabel.py | 25 +- .../nx_cugraph/tests/test_classes.py | 77 ++++ .../nx_cugraph/tests/test_ego_graph.py | 27 +- .../nx_cugraph/tests/test_generators.py | 35 +- .../nx_cugraph/tests/test_graph_methods.py | 4 +- .../nx_cugraph/tests/test_multigraph.py | 6 +- .../nx_cugraph/tests/testing_utils.py | 2 +- python/nx-cugraph/nx_cugraph/utils/misc.py | 2 +- python/nx-cugraph/run_nx_tests.sh | 6 +- 32 files changed, 848 insertions(+), 550 deletions(-) delete mode 100644 python/nx-cugraph/nx_cugraph/classes/zero.py create mode 100644 python/nx-cugraph/nx_cugraph/tests/test_classes.py diff --git a/ci/run_nx_cugraph_pytests.sh b/ci/run_nx_cugraph_pytests.sh index 52c6782c4ad..0e309d1e2d4 100755 --- a/ci/run_nx_cugraph_pytests.sh +++ b/ci/run_nx_cugraph_pytests.sh @@ -6,4 +6,5 @@ set -euo pipefail # Support invoking run_nx_cugraph_pytests.sh outside the script directory cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/nx-cugraph/nx_cugraph -NX_CUGRAPH_ZERO=False pytest --capture=no --cache-clear --benchmark-disable "$@" tests +NX_CUGRAPH_USE_COMPAT_GRAPHS=False pytest --capture=no --cache-clear --benchmark-disable "$@" tests +NX_CUGRAPH_USE_COMPAT_GRAPHS=True pytest --capture=no --cache-clear --benchmark-disable "$@" tests diff --git a/ci/test_python.sh b/ci/test_python.sh index 83138e96035..810284b8c97 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -108,7 +108,7 @@ echo "nx-cugraph coverage from networkx tests: $_coverage" echo $_coverage | awk '{ if ($NF == "0.0%") exit 1 }' # Ensure all algorithms were called by comparing covered lines to function lines. # Run our tests again (they're fast enough) to add their coverage, then create coverage.json -NX_CUGRAPH_ZERO=False pytest \ +NX_CUGRAPH_USE_COMPAT_GRAPHS=False pytest \ --pyargs nx_cugraph \ --config-file=../pyproject.toml \ --cov-config=../pyproject.toml \ diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index f93052a410b..e3690dfde6e 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -37,7 +37,7 @@ else DASK_DISTRIBUTED__SCHEDULER__WORKER_TTL="1000s" \ DASK_DISTRIBUTED__COMM__TIMEOUTS__CONNECT="1000s" \ DASK_CUDA_WAIT_WORKERS_MIN_TIMEOUT="1000s" \ - NX_CUGRAPH_ZERO=False \ + NX_CUGRAPH_USE_COMPAT_GRAPHS=False \ python -m pytest \ -v \ --import-mode=append \ diff --git a/python/nx-cugraph/_nx_cugraph/__init__.py b/python/nx-cugraph/_nx_cugraph/__init__.py index 4cb603f021d..428d266dd2e 100644 --- a/python/nx-cugraph/_nx_cugraph/__init__.py +++ b/python/nx-cugraph/_nx_cugraph/__init__.py @@ -296,7 +296,10 @@ def get_info(): del d[key] d["default_config"] = { - "zero": os.environ.get("NX_CUGRAPH_ZERO", "true").strip().lower() == "true", + "use_compat_graphs": os.environ.get("NX_CUGRAPH_USE_COMPAT_GRAPHS", "true") + .strip() + .lower() + == "true", } return d diff --git a/python/nx-cugraph/lint.yaml b/python/nx-cugraph/lint.yaml index b2184a185c4..f723eabb3e8 100644 --- a/python/nx-cugraph/lint.yaml +++ b/python/nx-cugraph/lint.yaml @@ -26,7 +26,7 @@ repos: - id: mixed-line-ending - id: trailing-whitespace - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.18 + rev: v0.19 hooks: - id: validate-pyproject name: Validate pyproject.toml @@ -40,29 +40,29 @@ repos: hooks: - id: isort - repo: https://github.com/asottile/pyupgrade - rev: v3.16.0 + rev: v3.17.0 hooks: - id: pyupgrade args: [--py310-plus] - repo: https://github.com/psf/black - rev: 24.4.2 + rev: 24.8.0 hooks: - id: black # - id: black-jupyter - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.4 + rev: v0.6.4 hooks: - id: ruff args: [--fix-only, --show-fixes] # --unsafe-fixes] - repo: https://github.com/PyCQA/flake8 - rev: 7.1.0 + rev: 7.1.1 hooks: - id: flake8 args: ['--per-file-ignores=_nx_cugraph/__init__.py:E501', '--extend-ignore=B020,SIM105'] # Why is this necessary? additional_dependencies: &flake8_dependencies # These versions need updated manually - - flake8==7.1.0 - - flake8-bugbear==24.4.26 + - flake8==7.1.1 + - flake8-bugbear==24.8.19 - flake8-simplify==0.21.0 - repo: https://github.com/asottile/yesqa rev: v1.5.0 @@ -77,7 +77,7 @@ repos: additional_dependencies: [tomli] files: ^(nx_cugraph|docs)/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.4 + rev: v0.6.4 hooks: - id: ruff - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/python/nx-cugraph/nx_cugraph/__init__.py b/python/nx-cugraph/nx_cugraph/__init__.py index 1c218458d6d..4404e57f645 100644 --- a/python/nx-cugraph/nx_cugraph/__init__.py +++ b/python/nx-cugraph/nx_cugraph/__init__.py @@ -39,8 +39,8 @@ from .interface import BackendInterface -BackendInterface.Graph = classes.ZeroGraph -BackendInterface.DiGraph = classes.ZeroDiGraph -BackendInterface.MultiGraph = classes.ZeroMultiGraph -BackendInterface.MultiDiGraph = classes.ZeroMultiDiGraph +BackendInterface.Graph = classes.Graph +BackendInterface.DiGraph = classes.DiGraph +BackendInterface.MultiGraph = classes.MultiGraph +BackendInterface.MultiDiGraph = classes.MultiDiGraph del BackendInterface diff --git a/python/nx-cugraph/nx_cugraph/algorithms/core.py b/python/nx-cugraph/nx_cugraph/algorithms/core.py index eff4c6aa4d5..e69ee88a17c 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/core.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/core.py @@ -59,10 +59,10 @@ def _(G): @networkx_algorithm(is_incomplete=True, version_added="23.12", _plc="k_truss_subgraph") def k_truss(G, k): if is_nx := isinstance(G, nx.Graph): - zero = isinstance(G, nxcg.ZeroGraph) + is_compat_graph = isinstance(G, nxcg.Graph) G = nxcg.from_networkx(G, preserve_all_attrs=True) else: - zero = False + is_compat_graph = False if nxcg.number_of_selfloops(G) > 0: if _nxver <= (3, 2): exc_class = nx.NetworkXError @@ -132,7 +132,7 @@ def k_truss(G, k): node_values, node_masks, key_to_id=key_to_id, - zero=zero, + use_compat_graph=is_compat_graph, ) new_graph.graph.update(G.graph) return new_graph diff --git a/python/nx-cugraph/nx_cugraph/algorithms/operators/unary.py b/python/nx-cugraph/nx_cugraph/algorithms/operators/unary.py index b09b6f7682a..75dc5fbc706 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/operators/unary.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/operators/unary.py @@ -23,7 +23,7 @@ @networkx_algorithm(version_added="24.02") def complement(G): - zero = isinstance(G, nxcg.ZeroGraph) + is_compat_graph = isinstance(G, nxcg.Graph) G = _to_graph(G) N = G._N # Upcast to int64 so indices don't overflow. @@ -44,7 +44,7 @@ def complement(G): src_indices.astype(index_dtype), dst_indices.astype(index_dtype), key_to_id=G.key_to_id, - zero=zero, + use_compat_graph=is_compat_graph, ) @@ -53,16 +53,16 @@ def reverse(G, copy=True): if not G.is_directed(): raise nx.NetworkXError("Cannot reverse an undirected graph.") if isinstance(G, nx.Graph): - zero = isinstance(G, nxcg.ZeroGraph) - if not copy and not zero: + is_compat_graph = isinstance(G, nxcg.Graph) + if not copy and not is_compat_graph: raise RuntimeError( "Using `copy=False` is invalid when using a NetworkX graph " "as input to `nx_cugraph.reverse`" ) G = nxcg.from_networkx(G, preserve_all_attrs=True) else: - zero = False + is_compat_graph = False rv = G.reverse(copy=copy) - if zero: - return rv.to_zero() + if is_compat_graph: + return rv._to_compat_graph() return rv diff --git a/python/nx-cugraph/nx_cugraph/algorithms/traversal/breadth_first_search.py b/python/nx-cugraph/nx_cugraph/algorithms/traversal/breadth_first_search.py index 1b1f048faaa..72d0079cf0c 100644 --- a/python/nx-cugraph/nx_cugraph/algorithms/traversal/breadth_first_search.py +++ b/python/nx-cugraph/nx_cugraph/algorithms/traversal/breadth_first_search.py @@ -133,15 +133,15 @@ def bfs_tree(G, source, reverse=False, depth_limit=None, sort_neighbors=None): raise NotImplementedError( "sort_neighbors argument in bfs_tree is not currently supported" ) - zero = isinstance(G, nxcg.ZeroGraph) + is_compat_graph = isinstance(G, nxcg.Graph) G = _check_G_and_source(G, source) if depth_limit is not None and depth_limit < 1: - return nxcg.DiGraph.from_coo( + return nxcg.CudaDiGraph.from_coo( 1, cp.array([], dtype=index_dtype), cp.array([], dtype=index_dtype), id_to_key=[source], - zero=zero, + use_compat_graph=is_compat_graph, ) distances, predecessors, node_ids = _bfs( @@ -151,12 +151,12 @@ def bfs_tree(G, source, reverse=False, depth_limit=None, sort_neighbors=None): reverse=reverse, ) if predecessors.size == 0: - return nxcg.DiGraph.from_coo( + return nxcg.CudaDiGraph.from_coo( 1, cp.array([], dtype=index_dtype), cp.array([], dtype=index_dtype), id_to_key=[source], - zero=zero, + use_compat_graph=is_compat_graph, ) # TODO: create renumbering helper function(s) unique_node_ids = cp.unique(cp.hstack((predecessors, node_ids))) @@ -174,12 +174,12 @@ def bfs_tree(G, source, reverse=False, depth_limit=None, sort_neighbors=None): old_index: new_index for new_index, old_index in enumerate(unique_node_ids.tolist()) } - return nxcg.DiGraph.from_coo( + return nxcg.CudaDiGraph.from_coo( unique_node_ids.size, src_indices, dst_indices, key_to_id=key_to_id, - zero=zero, + use_compat_graph=is_compat_graph, ) diff --git a/python/nx-cugraph/nx_cugraph/classes/__init__.py b/python/nx-cugraph/nx_cugraph/classes/__init__.py index f0ffcd2c850..71168e5364f 100644 --- a/python/nx-cugraph/nx_cugraph/classes/__init__.py +++ b/python/nx-cugraph/nx_cugraph/classes/__init__.py @@ -10,10 +10,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from .zero import ZeroGraph, ZeroDiGraph, ZeroMultiGraph, ZeroMultiDiGraph -from .graph import Graph -from .digraph import DiGraph -from .multigraph import MultiGraph -from .multidigraph import MultiDiGraph +from .graph import CudaGraph, Graph +from .digraph import CudaDiGraph, DiGraph +from .multigraph import CudaMultiGraph, MultiGraph +from .multidigraph import CudaMultiDiGraph, MultiDiGraph from .function import * diff --git a/python/nx-cugraph/nx_cugraph/classes/digraph.py b/python/nx-cugraph/nx_cugraph/classes/digraph.py index c51109796e7..178bf44f16e 100644 --- a/python/nx-cugraph/nx_cugraph/classes/digraph.py +++ b/python/nx-cugraph/nx_cugraph/classes/digraph.py @@ -18,38 +18,107 @@ import cupy as cp import networkx as nx import numpy as np +from networkx.classes.digraph import ( + _CachedPropertyResetterAdjAndSucc, + _CachedPropertyResetterPred, +) import nx_cugraph as nxcg from ..utils import index_dtype -from .graph import Graph -from .zero import ZeroDiGraph +from .graph import CudaGraph, Graph if TYPE_CHECKING: # pragma: no cover from nx_cugraph.typing import AttrKey -__all__ = ["DiGraph"] +__all__ = ["CudaDiGraph", "DiGraph"] networkx_api = nxcg.utils.decorators.networkx_class(nx.DiGraph) -class DiGraph(Graph): - ################# - # Class methods # - ################# +class DiGraph(nx.DiGraph, Graph): + _nx_attrs = ("_node", "_adj", "_succ", "_pred") + + name = Graph.name + _node = Graph._node + + @property + @networkx_api + def _adj(self): + if (adj := self.__dict__["_adj"]) is None: + self._reify_networkx() + adj = self.__dict__["_adj"] + return adj + + @_adj.setter + def _adj(self, val): + self._prepare_setter() + _CachedPropertyResetterAdjAndSucc.__set__(None, self, val) + if cache := getattr(self, "__networkx_cache__", None): + cache.clear() + + @property + @networkx_api + def _succ(self): + if (succ := self.__dict__["_succ"]) is None: + self._reify_networkx() + succ = self.__dict__["_succ"] + return succ + + @_succ.setter + def _succ(self, val): + self._prepare_setter() + _CachedPropertyResetterAdjAndSucc.__set__(None, self, val) + if cache := getattr(self, "__networkx_cache__", None): + cache.clear() + + @property + @networkx_api + def _pred(self): + if (pred := self.__dict__["_pred"]) is None: + self._reify_networkx() + pred = self.__dict__["_pred"] + return pred + + @_pred.setter + def _pred(self, val): + self._prepare_setter() + _CachedPropertyResetterPred.__set__(None, self, val) + if cache := getattr(self, "__networkx_cache__", None): + cache.clear() @classmethod @networkx_api def is_directed(cls) -> bool: return True + @classmethod + @networkx_api + def is_multigraph(cls) -> bool: + return False + + @classmethod + def to_cudagraph_class(cls) -> type[CudaDiGraph]: + return CudaDiGraph + @classmethod def to_networkx_class(cls) -> type[nx.DiGraph]: return nx.DiGraph + +class CudaDiGraph(CudaGraph): + ################# + # Class methods # + ################# + + is_directed = classmethod(DiGraph.is_directed.__func__) + is_multigraph = classmethod(DiGraph.is_multigraph.__func__) + to_cudagraph_class = classmethod(DiGraph.to_cudagraph_class.__func__) + to_networkx_class = classmethod(DiGraph.to_networkx_class.__func__) + @classmethod - def to_zero_class(cls) -> type[ZeroDiGraph]: - return ZeroDiGraph + def _to_compat_graph_class(cls) -> type[DiGraph]: + return DiGraph @networkx_api def size(self, weight: AttrKey | None = None) -> int: @@ -62,7 +131,7 @@ def size(self, weight: AttrKey | None = None) -> int: ########################## @networkx_api - def reverse(self, copy: bool = True) -> DiGraph: + def reverse(self, copy: bool = True) -> CudaDiGraph: return self._copy(not copy, self.__class__, reverse=True) @networkx_api @@ -167,7 +236,7 @@ def to_undirected(self, reciprocal=False, as_view=False): node_masks, key_to_id=key_to_id, id_to_key=id_to_key, - zero=False, + use_compat_graph=False, ) if as_view: rv.graph = self.graph diff --git a/python/nx-cugraph/nx_cugraph/classes/graph.py b/python/nx-cugraph/nx_cugraph/classes/graph.py index 2abf0cd6502..78d74f2d384 100644 --- a/python/nx-cugraph/nx_cugraph/classes/graph.py +++ b/python/nx-cugraph/nx_cugraph/classes/graph.py @@ -20,12 +20,15 @@ import networkx as nx import numpy as np import pylibcugraph as plc +from networkx.classes.graph import ( + _CachedPropertyResetterAdj, + _CachedPropertyResetterNode, +) import nx_cugraph as nxcg from nx_cugraph import _nxver from ..utils import index_dtype -from .zero import ZeroGraph if TYPE_CHECKING: # pragma: no cover from collections.abc import Iterable, Iterator @@ -42,57 +45,236 @@ any_ndarray, ) -__all__ = ["Graph"] +__all__ = ["CudaGraph", "Graph"] networkx_api = nxcg.utils.decorators.networkx_class(nx.Graph) +if _nxver < (3, 4): + _CACHE_KEY = (True, True, True) +else: + _CACHE_KEY = (True, True) -class Graph: +# Use to indicate when a full conversion to GPU failed so we don't try again. +_CANT_CONVERT_TO_GPU = "_CANT_CONVERT_TO_GPU" + + +# `collections.UserDict` was the preferred way to subclass dict, but now +# subclassing dict directly is much better supported and should work here. +# This class should only be necessary if the user clears the cache manually. +class _GraphCache(dict): + """Cache that ensures Graph will reify into a NetworkX graph when cleared.""" + + _graph: Graph + + def __init__(self, graph: Graph): + self._graph = graph + + def clear(self) -> None: + self._graph._reify_networkx() + super().clear() + + +class Graph(nx.Graph): # Tell networkx to dispatch calls with this object to nx-cugraph __networkx_backend__: ClassVar[str] = "cugraph" # nx >=3.2 __networkx_plugin__: ClassVar[str] = "cugraph" # nx <3.2 + # Core attributes of NetowkrX graphs that will be copied and cleared as appropriate. + # These attributes comprise the edge and node data model for NetworkX graphs. + _nx_attrs = ("_node", "_adj") + # Allow networkx dispatch machinery to cache conversions. # This means we should clear the cache if we ever mutate the object! - __networkx_cache__: dict | None + __networkx_cache__: _GraphCache | None # networkx properties graph: dict - graph_attr_dict_factory: ClassVar[type] = dict + # Should we declare type annotations for the rest? - # Not networkx properties - # We store edge data in COO format with {src,dst}_indices and edge_values. - src_indices: cp.ndarray[IndexValue] - dst_indices: cp.ndarray[IndexValue] - edge_values: dict[AttrKey, cp.ndarray[EdgeValue]] - edge_masks: dict[AttrKey, cp.ndarray[bool]] - node_values: dict[AttrKey, any_ndarray[NodeValue]] - node_masks: dict[AttrKey, any_ndarray[bool]] - key_to_id: dict[NodeKey, IndexValue] | None - _id_to_key: list[NodeKey] | None - _N: int - _node_ids: cp.ndarray[IndexValue] | None # holds plc.SGGraph.vertices_array data + # Properties that trigger copying to the CPU + def _prepare_setter(self): + """Be careful when setting private attributes which may be used during init.""" + if ( + # If not present, then this must be in init + any(attr not in self.__dict__ for attr in self._nx_attrs) + # Already on the CPU + or not any(self.__dict__[attr] is None for attr in self._nx_attrs) + ): + return + if self._is_on_gpu: + # Copy from GPU to CPU + self._reify_networkx() + return + # Default values + for attr in self._nx_attrs: + if self.__dict__[attr] is None: + if attr == "_succ": + self.__dict__[attr] = self.__dict__["_adj"] + else: + self.__dict__[attr] = {} - # Used by graph._get_plc_graph - _plc_type_map: ClassVar[dict[np.dtype, np.dtype]] = { - # signed int - np.dtype(np.int8): np.dtype(np.float32), - np.dtype(np.int16): np.dtype(np.float32), - np.dtype(np.int32): np.dtype(np.float64), - np.dtype(np.int64): np.dtype(np.float64), # raise if abs(x) > 2**53 - # unsigned int - np.dtype(np.uint8): np.dtype(np.float32), - np.dtype(np.uint16): np.dtype(np.float32), - np.dtype(np.uint32): np.dtype(np.float64), - np.dtype(np.uint64): np.dtype(np.float64), # raise if x > 2**53 - # other - np.dtype(np.bool_): np.dtype(np.float32), - np.dtype(np.float16): np.dtype(np.float32), - } - _plc_allowed_edge_types: ClassVar[set[np.dtype]] = { - np.dtype(np.float32), - np.dtype(np.float64), - } + @property + @networkx_api + def _node(self): + if (node := self.__dict__["_node"]) is None: + self._reify_networkx() + node = self.__dict__["_node"] + return node + + @_node.setter + def _node(self, val): + self._prepare_setter() + _CachedPropertyResetterNode.__set__(None, self, val) + if cache := getattr(self, "__networkx_cache__", None): + cache.clear() + + @property + @networkx_api + def _adj(self): + if (adj := self.__dict__["_adj"]) is None: + self._reify_networkx() + adj = self.__dict__["_adj"] + return adj + + @_adj.setter + def _adj(self, val): + self._prepare_setter() + _CachedPropertyResetterAdj.__set__(None, self, val) + if cache := getattr(self, "__networkx_cache__", None): + cache.clear() + + @property + def _is_on_gpu(self) -> bool: + """Whether the full graph is on device (in the cache). + + This returns False when only a subset of the graph (such as only + edge indices and edge attribute) is on device. + + The graph may be on host (CPU) and device (GPU) at the same time. + """ + cache = getattr(self, "__networkx_cache__", None) + if not cache: + return False + return _CACHE_KEY in cache.get("backends", {}).get("cugraph", {}) + + @property + def _is_on_cpu(self) -> bool: + """Whether the graph is on host as a NetworkX graph. + + This means the core data structures that comprise a NetworkX graph + (such as ``G._node`` and ``G._adj``) are present. + + The graph may be on host (CPU) and device (GPU) at the same time. + """ + return self.__dict__["_node"] is not None + + @property + def _cudagraph(self): + """Return the full ``CudaGraph`` on device, computing if necessary, or None.""" + nx_cache = getattr(self, "__networkx_cache__", None) + if nx_cache is None: + nx_cache = {} + elif _CANT_CONVERT_TO_GPU in nx_cache: + return None + cache = nx_cache.setdefault("backends", {}).setdefault("cugraph", {}) + if (Gcg := cache.get(_CACHE_KEY)) is not None: + if isinstance(Gcg, Graph): + # This shouldn't happen during normal use, but be extra-careful anyway + return Gcg._cudagraph + return Gcg + if self.__dict__["_node"] is None: + raise RuntimeError( + f"{type(self).__name__} cannot be converted to the GPU, because it is " + "not on the CPU! This is not supposed to be possible. If you believe " + "you have found a bug, please report a minimum reproducible example to " + "https://github.com/rapidsai/cugraph/issues/new/choose" + ) + try: + Gcg = nxcg.from_networkx( + self, preserve_edge_attrs=True, preserve_node_attrs=True + ) + except Exception: + # Should we warn that the full graph can't be on GPU? + nx_cache[_CANT_CONVERT_TO_GPU] = True + return None + Gcg.graph = self.graph + cache[_CACHE_KEY] = Gcg + return Gcg + + @_cudagraph.setter + def _cudagraph(self, val, *, clear_cpu=True): + """Set the full ``CudaGraph`` for this graph, or remove from device if None.""" + if (cache := getattr(self, "__networkx_cache__", None)) is None: + # Should we warn? + return + # TODO: pay close attention to when we should clear the cache, since + # this may or may not be a mutation. + cache = cache.setdefault("backends", {}).setdefault("cugraph", {}) + if val is None: + cache.pop(_CACHE_KEY, None) + else: + self.graph = val.graph + cache[_CACHE_KEY] = val + if clear_cpu: + for key in self._nx_attrs: + self.__dict__[key] = None + + @nx.Graph.name.setter + def name(self, s): + # Don't clear the cache when setting the name, since `.graph` is shared. + # There is a very small risk here for the cache to become (slightly) + # insconsistent if graphs from other backends are cached. + self.graph["name"] = s + + @classmethod + @networkx_api + def is_directed(cls) -> bool: + return False + + @classmethod + @networkx_api + def is_multigraph(cls) -> bool: + return False + + @classmethod + def to_cudagraph_class(cls) -> type[CudaGraph]: + return CudaGraph + + @classmethod + @networkx_api + def to_directed_class(cls) -> type[nxcg.DiGraph]: + return nxcg.DiGraph + + @classmethod + def to_networkx_class(cls) -> type[nx.Graph]: + return nx.Graph + + @classmethod + @networkx_api + def to_undirected_class(cls) -> type[Graph]: + return Graph + + def __init__(self, incoming_graph_data=None, **attr): + super().__init__(incoming_graph_data, **attr) + self.__networkx_cache__ = _GraphCache(self) + + def _reify_networkx(self) -> None: + """Copy graph to host (CPU) if necessary.""" + if self.__dict__["_node"] is None: + # After we make this into an nx graph, we rely on the cache being correct + Gcg = self._cudagraph + G = nxcg.to_networkx(Gcg) + for key in self._nx_attrs: + self.__dict__[key] = G.__dict__[key] + + def _become(self, other: Graph): + if self.__class__ is not other.__class__: + raise TypeError( + "Attempting to update graph inplace with graph of different type!" + ) + # Begin with the simplest implementation; do we need to do more? + self.__dict__.update(other.__dict__) + return self #################### # Creation methods # @@ -111,10 +293,10 @@ def from_coo( *, key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, - zero: bool | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> Graph: - new_graph = object.__new__(cls) + ) -> Graph | CudaGraph: + new_graph = object.__new__(cls.to_cudagraph_class()) new_graph.__networkx_cache__ = {} new_graph.src_indices = src_indices new_graph.dst_indices = dst_indices @@ -176,13 +358,8 @@ def from_coo( isolates = nxcg.algorithms.isolate._isolates(new_graph) if len(isolates) > 0: new_graph._node_ids = cp.arange(new_graph._N, dtype=index_dtype) - if ( - zero - or zero is None - and _nxver >= (3, 3) - and nx.config.backends.cugraph.zero - ): - new_graph = new_graph.to_zero() + if use_compat_graph or use_compat_graph is None and issubclass(cls, Graph): + new_graph = new_graph._to_compat_graph() return new_graph @classmethod @@ -197,9 +374,9 @@ def from_csr( *, key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, - zero: bool | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> Graph: + ) -> Graph | CudaGraph: N = indptr.size - 1 src_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead @@ -215,7 +392,7 @@ def from_csr( node_masks, key_to_id=key_to_id, id_to_key=id_to_key, - zero=zero, + use_compat_graph=use_compat_graph, **attr, ) @@ -231,9 +408,9 @@ def from_csc( *, key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, - zero: bool | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> Graph: + ) -> Graph | CudaGraph: N = indptr.size - 1 dst_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead @@ -249,7 +426,7 @@ def from_csc( node_masks, key_to_id=key_to_id, id_to_key=id_to_key, - zero=zero, + use_compat_graph=use_compat_graph, **attr, ) @@ -267,9 +444,9 @@ def from_dcsr( *, key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, - zero: bool | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> Graph: + ) -> Graph | CudaGraph: src_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead np.repeat(compressed_srcs.get(), cp.diff(indptr).get()) @@ -284,7 +461,7 @@ def from_dcsr( node_masks, key_to_id=key_to_id, id_to_key=id_to_key, - zero=zero, + use_compat_graph=use_compat_graph, **attr, ) @@ -302,9 +479,9 @@ def from_dcsc( *, key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, - zero: bool | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> Graph: + ) -> Graph | CudaGraph: dst_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead np.repeat(compressed_dsts.get(), cp.diff(indptr).get()) @@ -319,14 +496,75 @@ def from_dcsc( node_masks, key_to_id=key_to_id, id_to_key=id_to_key, - zero=zero, + use_compat_graph=use_compat_graph, **attr, ) - def __new__(cls, incoming_graph_data=None, **attr) -> Graph: + +class CudaGraph: + # Tell networkx to dispatch calls with this object to nx-cugraph + __networkx_backend__: ClassVar[str] = "cugraph" # nx >=3.2 + __networkx_plugin__: ClassVar[str] = "cugraph" # nx <3.2 + + # Allow networkx dispatch machinery to cache conversions. + # This means we should clear the cache if we ever mutate the object! + __networkx_cache__: dict | None + + # networkx properties + graph: dict + graph_attr_dict_factory: ClassVar[type] = dict + + # Not networkx properties + # We store edge data in COO format with {src,dst}_indices and edge_values. + src_indices: cp.ndarray[IndexValue] + dst_indices: cp.ndarray[IndexValue] + edge_values: dict[AttrKey, cp.ndarray[EdgeValue]] + edge_masks: dict[AttrKey, cp.ndarray[bool]] + node_values: dict[AttrKey, any_ndarray[NodeValue]] + node_masks: dict[AttrKey, any_ndarray[bool]] + key_to_id: dict[NodeKey, IndexValue] | None + _id_to_key: list[NodeKey] | None + _N: int + _node_ids: cp.ndarray[IndexValue] | None # holds plc.SGGraph.vertices_array data + + # Used by graph._get_plc_graph + _plc_type_map: ClassVar[dict[np.dtype, np.dtype]] = { + # signed int + np.dtype(np.int8): np.dtype(np.float32), + np.dtype(np.int16): np.dtype(np.float32), + np.dtype(np.int32): np.dtype(np.float64), + np.dtype(np.int64): np.dtype(np.float64), # raise if abs(x) > 2**53 + # unsigned int + np.dtype(np.uint8): np.dtype(np.float32), + np.dtype(np.uint16): np.dtype(np.float32), + np.dtype(np.uint32): np.dtype(np.float64), + np.dtype(np.uint64): np.dtype(np.float64), # raise if x > 2**53 + # other + np.dtype(np.bool_): np.dtype(np.float32), + np.dtype(np.float16): np.dtype(np.float32), + } + _plc_allowed_edge_types: ClassVar[set[np.dtype]] = { + np.dtype(np.float32), + np.dtype(np.float64), + } + + #################### + # Creation methods # + #################### + + from_coo = classmethod(Graph.from_coo.__func__) + from_csr = classmethod(Graph.from_csr.__func__) + from_csc = classmethod(Graph.from_csc.__func__) + from_dcsr = classmethod(Graph.from_dcsr.__func__) + from_dcsc = classmethod(Graph.from_dcsc.__func__) + + def __new__(cls, incoming_graph_data=None, **attr) -> CudaGraph: if incoming_graph_data is None: new_graph = cls.from_coo( - 0, cp.empty(0, index_dtype), cp.empty(0, index_dtype), zero=False + 0, + cp.empty(0, index_dtype), + cp.empty(0, index_dtype), + use_compat_graph=False, ) elif incoming_graph_data.__class__ is cls: new_graph = incoming_graph_data.copy() @@ -335,41 +573,32 @@ def __new__(cls, incoming_graph_data=None, **attr) -> Graph: else: raise NotImplementedError new_graph.graph.update(attr) - # We could return ZeroGraph here (if configured), but let's not for now + # We could return Graph here (if configured), but let's not for now return new_graph ################# # Class methods # ################# - @classmethod - @networkx_api - def is_directed(cls) -> bool: - return False + is_directed = classmethod(Graph.is_directed.__func__) + is_multigraph = classmethod(Graph.is_multigraph.__func__) + to_cudagraph_class = classmethod(Graph.to_cudagraph_class.__func__) + to_networkx_class = classmethod(Graph.to_networkx_class.__func__) @classmethod @networkx_api - def is_multigraph(cls) -> bool: - return False + def to_directed_class(cls) -> type[nxcg.CudaDiGraph]: + return nxcg.CudaDiGraph @classmethod @networkx_api - def to_directed_class(cls) -> type[nxcg.DiGraph]: - return nxcg.DiGraph - - @classmethod - def to_networkx_class(cls) -> type[nx.Graph]: - return nx.Graph + def to_undirected_class(cls) -> type[CudaGraph]: + return CudaGraph @classmethod - @networkx_api - def to_undirected_class(cls) -> type[Graph]: + def _to_compat_graph_class(cls) -> type[Graph]: return Graph - @classmethod - def to_zero_class(cls) -> type[ZeroGraph]: - return ZeroGraph - ############## # Properties # ############## @@ -460,7 +689,7 @@ def clear_edges(self) -> None: cache.clear() @networkx_api - def copy(self, as_view: bool = False) -> Graph: + def copy(self, as_view: bool = False) -> CudaGraph: # Does shallow copy in networkx return self._copy(as_view, self.__class__) @@ -556,17 +785,17 @@ def size(self, weight: AttrKey | None = None) -> int: return int(cp.count_nonzero(self.src_indices <= self.dst_indices)) @networkx_api - def to_directed(self, as_view: bool = False) -> nxcg.DiGraph: + def to_directed(self, as_view: bool = False) -> nxcg.CudaDiGraph: return self._copy(as_view, self.to_directed_class()) @networkx_api - def to_undirected(self, as_view: bool = False) -> Graph: + def to_undirected(self, as_view: bool = False) -> CudaGraph: # Does deep copy in networkx return self._copy(as_view, self.to_undirected_class()) - def to_zero(self) -> ZeroGraph: - rv = self.to_zero_class()() - rv._cugraph = self + def _to_compat_graph(self) -> Graph: + rv = self._to_compat_graph_class()() + rv._cudagraph = self return rv # Not implemented... @@ -579,8 +808,8 @@ def to_zero(self) -> ZeroGraph: # Private methods # ################### - def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): - # DRY warning: see also MultiGraph._copy + def _copy(self, as_view: bool, cls: type[CudaGraph], reverse: bool = False): + # DRY warning: see also CudaMultiGraph._copy src_indices = self.src_indices dst_indices = self.dst_indices edge_values = self.edge_values @@ -620,7 +849,7 @@ def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): node_masks, key_to_id=key_to_id, id_to_key=id_to_key, - zero=False, + use_compat_graph=False, ) if as_view: rv.graph = self.graph @@ -733,7 +962,7 @@ def _get_plc_graph( ) def _sort_edge_indices(self, primary="src"): - # DRY warning: see also MultiGraph._sort_edge_indices + # DRY warning: see also CudaMultiGraph._sort_edge_indices if primary == "src": stacked = cp.vstack((self.dst_indices, self.src_indices)) elif primary == "dst": @@ -755,7 +984,7 @@ def _sort_edge_indices(self, primary="src"): {key: val[indices] for key, val in self.edge_masks.items()} ) - def _become(self, other: Graph): + def _become(self, other: CudaGraph): if self.__class__ is not other.__class__: raise TypeError( "Attempting to update graph inplace with graph of different type!" diff --git a/python/nx-cugraph/nx_cugraph/classes/multidigraph.py b/python/nx-cugraph/nx_cugraph/classes/multidigraph.py index 93deff8b231..5a6595567d2 100644 --- a/python/nx-cugraph/nx_cugraph/classes/multidigraph.py +++ b/python/nx-cugraph/nx_cugraph/classes/multidigraph.py @@ -16,28 +16,50 @@ import nx_cugraph as nxcg -from .digraph import DiGraph -from .multigraph import MultiGraph -from .zero import ZeroMultiDiGraph +from .digraph import CudaDiGraph, DiGraph +from .graph import Graph +from .multigraph import CudaMultiGraph, MultiGraph -__all__ = ["MultiDiGraph"] +__all__ = ["CudaMultiDiGraph", "MultiDiGraph"] networkx_api = nxcg.utils.decorators.networkx_class(nx.MultiDiGraph) -class MultiDiGraph(MultiGraph, DiGraph): +class MultiDiGraph(nx.MultiDiGraph, MultiGraph, DiGraph): + name = Graph.name + _node = Graph._node + _adj = DiGraph._adj + _succ = DiGraph._succ + _pred = DiGraph._pred + @classmethod @networkx_api def is_directed(cls) -> bool: return True + @classmethod + @networkx_api + def is_multigraph(cls) -> bool: + return True + + @classmethod + def to_cudagraph_class(cls) -> type[CudaMultiDiGraph]: + return CudaMultiDiGraph + @classmethod def to_networkx_class(cls) -> type[nx.MultiDiGraph]: return nx.MultiDiGraph + +class CudaMultiDiGraph(CudaMultiGraph, CudaDiGraph): + is_directed = classmethod(MultiDiGraph.is_directed.__func__) + is_multigraph = classmethod(MultiDiGraph.is_multigraph.__func__) + to_cudagraph_class = classmethod(MultiDiGraph.to_cudagraph_class.__func__) + to_networkx_class = classmethod(MultiDiGraph.to_networkx_class.__func__) + @classmethod - def to_zero_class(cls) -> type[ZeroMultiDiGraph]: - return ZeroMultiDiGraph + def _to_compat_graph_class(cls) -> type[MultiDiGraph]: + return MultiDiGraph ########################## # NetworkX graph methods # diff --git a/python/nx-cugraph/nx_cugraph/classes/multigraph.py b/python/nx-cugraph/nx_cugraph/classes/multigraph.py index 2c5083256d2..c8c8f1dfb00 100644 --- a/python/nx-cugraph/nx_cugraph/classes/multigraph.py +++ b/python/nx-cugraph/nx_cugraph/classes/multigraph.py @@ -20,11 +20,9 @@ import numpy as np import nx_cugraph as nxcg -from nx_cugraph import _nxver from ..utils import index_dtype -from .graph import Graph -from .zero import ZeroMultiGraph +from .graph import CudaGraph, Graph, _GraphCache if TYPE_CHECKING: from nx_cugraph.typing import ( @@ -36,32 +34,47 @@ NodeValue, any_ndarray, ) -__all__ = ["MultiGraph"] +__all__ = ["MultiGraph", "CudaMultiGraph"] networkx_api = nxcg.utils.decorators.networkx_class(nx.MultiGraph) -class MultiGraph(Graph): - # networkx properties - edge_key_dict_factory: ClassVar[type] = dict +class MultiGraph(nx.MultiGraph, Graph): + name = Graph.name + _node = Graph._node + _adj = Graph._adj - # Not networkx properties + @classmethod + @networkx_api + def is_directed(cls) -> bool: + return False - # In a MultiGraph, each edge has a unique `(src, dst, key)` key. - # By default, `key` is 0 if possible, else 1, else 2, etc. - # This key can be any hashable Python object in NetworkX. - # We don't use a dict for our data structure here, because - # that would require a `(src, dst, key)` key. - # Instead, we keep `edge_keys` and/or `edge_indices`. - # `edge_keys` is the list of Python objects for each edge. - # `edge_indices` is for the common case of default multiedge keys, - # in which case we can store it as a cupy array. - # `edge_indices` is generally preferred. It is possible to provide - # both where edge_indices is the default and edge_keys is anything. - # It is also possible for them both to be None, which means the - # default edge indices has not yet been calculated. - edge_indices: cp.ndarray[IndexValue] | None - edge_keys: list[EdgeKey] | None + @classmethod + @networkx_api + def is_multigraph(cls) -> bool: + return True + + @classmethod + def to_cudagraph_class(cls) -> type[CudaMultiGraph]: + return CudaMultiGraph + + @classmethod + @networkx_api + def to_directed_class(cls) -> type[nxcg.MultiDiGraph]: + return nxcg.MultiDiGraph + + @classmethod + def to_networkx_class(cls) -> type[nx.MultiGraph]: + return nx.MultiGraph + + @classmethod + @networkx_api + def to_undirected_class(cls) -> type[MultiGraph]: + return MultiGraph + + def __init__(self, incoming_graph_data=None, multigraph_input=None, **attr): + super().__init__(incoming_graph_data, multigraph_input, **attr) + self.__networkx_cache__ = _GraphCache(self) #################### # Creation methods # @@ -82,10 +95,10 @@ def from_coo( key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, edge_keys: list[EdgeKey] | None = None, - zero: bool | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> MultiGraph: - new_graph = super().from_coo( + ) -> MultiGraph | CudaMultiGraph: + new_graph = super(cls.to_undirected_class(), cls).from_coo( N, src_indices, dst_indices, @@ -95,7 +108,7 @@ def from_coo( node_masks, key_to_id=key_to_id, id_to_key=id_to_key, - zero=False, + use_compat_graph=False, **attr, ) new_graph.edge_indices = edge_indices @@ -106,13 +119,8 @@ def from_coo( and len(new_graph.edge_keys) != src_indices.size ): raise ValueError - if ( - zero - or zero is None - and _nxver >= (3, 3) - and nx.config.backends.cugraph.zero - ): - new_graph = new_graph.to_zero() + if use_compat_graph or use_compat_graph is None and issubclass(cls, Graph): + new_graph = new_graph._to_compat_graph() return new_graph @classmethod @@ -129,9 +137,9 @@ def from_csr( key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, edge_keys: list[EdgeKey] | None = None, - zero: bool | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> MultiGraph: + ) -> MultiGraph | CudaMultiGraph: N = indptr.size - 1 src_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead @@ -149,7 +157,7 @@ def from_csr( key_to_id=key_to_id, id_to_key=id_to_key, edge_keys=edge_keys, - zero=zero, + use_compat_graph=use_compat_graph, **attr, ) @@ -167,9 +175,9 @@ def from_csc( key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, edge_keys: list[EdgeKey] | None = None, - zero: bool | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> MultiGraph: + ) -> MultiGraph | CudaMultiGraph: N = indptr.size - 1 dst_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead @@ -187,7 +195,7 @@ def from_csc( key_to_id=key_to_id, id_to_key=id_to_key, edge_keys=edge_keys, - zero=zero, + use_compat_graph=use_compat_graph, **attr, ) @@ -207,9 +215,9 @@ def from_dcsr( key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, edge_keys: list[EdgeKey] | None = None, - zero: bool | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> MultiGraph: + ) -> MultiGraph | CudaMultiGraph: src_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead np.repeat(compressed_srcs.get(), cp.diff(indptr).get()) @@ -226,7 +234,7 @@ def from_dcsr( key_to_id=key_to_id, id_to_key=id_to_key, edge_keys=edge_keys, - zero=zero, + use_compat_graph=use_compat_graph, **attr, ) @@ -246,9 +254,9 @@ def from_dcsc( key_to_id: dict[NodeKey, IndexValue] | None = None, id_to_key: list[NodeKey] | None = None, edge_keys: list[EdgeKey] | None = None, - zero: bool | None = None, + use_compat_graph: bool | None = None, **attr, - ) -> Graph: + ) -> MultiGraph | CudaGraph: dst_indices = cp.array( # cp.repeat is slow to use here, so use numpy instead np.repeat(compressed_dsts.get(), cp.diff(indptr).get()) @@ -265,13 +273,46 @@ def from_dcsc( key_to_id=key_to_id, id_to_key=id_to_key, edge_keys=edge_keys, - zero=zero, + use_compat_graph=use_compat_graph, **attr, ) + +class CudaMultiGraph(CudaGraph): + # networkx properties + edge_key_dict_factory: ClassVar[type] = dict + + # Not networkx properties + + # In a MultiGraph, each edge has a unique `(src, dst, key)` key. + # By default, `key` is 0 if possible, else 1, else 2, etc. + # This key can be any hashable Python object in NetworkX. + # We don't use a dict for our data structure here, because + # that would require a `(src, dst, key)` key. + # Instead, we keep `edge_keys` and/or `edge_indices`. + # `edge_keys` is the list of Python objects for each edge. + # `edge_indices` is for the common case of default multiedge keys, + # in which case we can store it as a cupy array. + # `edge_indices` is generally preferred. It is possible to provide + # both where edge_indices is the default and edge_keys is anything. + # It is also possible for them both to be None, which means the + # default edge indices has not yet been calculated. + edge_indices: cp.ndarray[IndexValue] | None + edge_keys: list[EdgeKey] | None + + #################### + # Creation methods # + #################### + + from_coo = classmethod(MultiGraph.from_coo.__func__) + from_csr = classmethod(MultiGraph.from_csr.__func__) + from_csc = classmethod(MultiGraph.from_csc.__func__) + from_dcsr = classmethod(MultiGraph.from_dcsr.__func__) + from_dcsc = classmethod(MultiGraph.from_dcsc.__func__) + def __new__( cls, incoming_graph_data=None, multigraph_input=None, **attr - ) -> MultiGraph: + ) -> CudaMultiGraph: if isinstance(incoming_graph_data, dict) and multigraph_input is not False: new_graph = nxcg.from_networkx( nx.MultiGraph(incoming_graph_data, multigraph_input=multigraph_input), @@ -286,34 +327,25 @@ def __new__( # Class methods # ################# - @classmethod - @networkx_api - def is_directed(cls) -> bool: - return False + is_directed = classmethod(MultiGraph.is_directed.__func__) + is_multigraph = classmethod(MultiGraph.is_multigraph.__func__) + to_cudagraph_class = classmethod(MultiGraph.to_cudagraph_class.__func__) + to_networkx_class = classmethod(MultiGraph.to_networkx_class.__func__) @classmethod @networkx_api - def is_multigraph(cls) -> bool: - return True + def to_directed_class(cls) -> type[nxcg.CudaMultiDiGraph]: + return nxcg.CudaMultiDiGraph @classmethod @networkx_api - def to_directed_class(cls) -> type[nxcg.MultiDiGraph]: - return nxcg.MultiDiGraph + def to_undirected_class(cls) -> type[CudaMultiGraph]: + return CudaMultiGraph @classmethod - def to_networkx_class(cls) -> type[nx.MultiGraph]: - return nx.MultiGraph - - @classmethod - @networkx_api - def to_undirected_class(cls) -> type[MultiGraph]: + def _to_compat_graph_class(cls) -> type[MultiGraph]: return MultiGraph - @classmethod - def to_zero_class(cls) -> type[ZeroMultiGraph]: - return ZeroMultiGraph - ########################## # NetworkX graph methods # ########################## @@ -331,7 +363,7 @@ def clear_edges(self) -> None: self.edge_keys = None @networkx_api - def copy(self, as_view: bool = False) -> MultiGraph: + def copy(self, as_view: bool = False) -> CudaMultiGraph: # Does shallow copy in networkx return self._copy(as_view, self.__class__) @@ -414,11 +446,11 @@ def has_edge(self, u: NodeKey, v: NodeKey, key: EdgeKey | None = None) -> bool: return any(edge_keys[i] == key for i in indices.tolist()) @networkx_api - def to_directed(self, as_view: bool = False) -> nxcg.MultiDiGraph: + def to_directed(self, as_view: bool = False) -> nxcg.CudaMultiDiGraph: return self._copy(as_view, self.to_directed_class()) @networkx_api - def to_undirected(self, as_view: bool = False) -> MultiGraph: + def to_undirected(self, as_view: bool = False) -> CudaMultiGraph: # Does deep copy in networkx return self._copy(as_view, self.to_undirected_class()) @@ -426,8 +458,8 @@ def to_undirected(self, as_view: bool = False) -> MultiGraph: # Private methods # ################### - def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): - # DRY warning: see also Graph._copy + def _copy(self, as_view: bool, cls: type[CudaGraph], reverse: bool = False): + # DRY warning: see also CudaGraph._copy src_indices = self.src_indices dst_indices = self.dst_indices edge_indices = self.edge_indices @@ -474,7 +506,7 @@ def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): key_to_id=key_to_id, id_to_key=id_to_key, edge_keys=edge_keys, - zero=False, + use_compat_graph=False, ) if as_view: rv.graph = self.graph @@ -484,7 +516,7 @@ def _copy(self, as_view: bool, cls: type[Graph], reverse: bool = False): return rv def _sort_edge_indices(self, primary="src"): - # DRY warning: see also Graph._sort_edge_indices + # DRY warning: see also CudaGraph._sort_edge_indices if self.edge_indices is None and self.edge_keys is None: return super()._sort_edge_indices(primary=primary) if primary == "src": diff --git a/python/nx-cugraph/nx_cugraph/classes/zero.py b/python/nx-cugraph/nx_cugraph/classes/zero.py deleted file mode 100644 index d3968766816..00000000000 --- a/python/nx-cugraph/nx_cugraph/classes/zero.py +++ /dev/null @@ -1,240 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import networkx as nx -from networkx.classes.digraph import ( - _CachedPropertyResetterAdjAndSucc, - _CachedPropertyResetterPred, -) -from networkx.classes.graph import ( - _CachedPropertyResetterAdj, - _CachedPropertyResetterNode, -) - -import nx_cugraph as nxcg -from nx_cugraph import _nxver - -if _nxver < (3, 4): - _CACHE_KEY = (True, True, True) -else: - _CACHE_KEY = (True, True) - -# Use to indicate when a full conversion to GPU failed so we don't try again. -_CANT_CONVERT_TO_GPU = "_CANT_CONVERT_TO_GPU" - - -# `collections.UserDict` was the preferred way to subclass dict, but now -# subclassing dict directly is much better supported and should work here. -# This class should only be necessary if the user clears the cache manually. -class _ZeroGraphCache(dict): - """Cache that ensures ZeroGraph will reify into a NetworkX graph when cleared.""" - - def __init__(self, zero_graph): - self._zero_graph = zero_graph - - def clear(self): - self._zero_graph._reify_networkx() - super().clear() - - -# TODO: experiment whether we should make `__class__` and other attrs match networkx -class ZeroGraph(nx.Graph): - __networkx_backend__ = "cugraph" - - _nx_attrs = ("_node", "_adj") - - @property - def _node(self): - if (node := self.__dict__["_node"]) is None: - self._reify_networkx() - node = self.__dict__["_node"] - return node - - @_node.setter - def _node(self, val): - _CachedPropertyResetterNode.__set__(None, self, val) - - @property - def _adj(self): - if (adj := self.__dict__["_adj"]) is None: - self._reify_networkx() - adj = self.__dict__["_adj"] - return adj - - @_adj.setter - def _adj(self, val): - _CachedPropertyResetterAdj.__set__(None, self, val) - - @property - def _is_on_gpu(self): - cache = getattr(self, "__networkx_cache__", None) - if not cache: - return False - return _CACHE_KEY in cache.get("backends", {}).get("cugraph", {}) - - @property - def _is_on_cpu(self): - return self.__dict__["_node"] is not None - - @property - def _cugraph(self): - nx_cache = getattr(self, "__networkx_cache__", None) - if nx_cache is None: - nx_cache = {} - elif _CANT_CONVERT_TO_GPU in nx_cache: - return None - cache = nx_cache.setdefault("backends", {}).setdefault("cugraph", {}) - if (Gcg := cache.get(_CACHE_KEY)) is not None: - return Gcg - if self.__dict__["_node"] is None: - raise RuntimeError( - f"{type(self).__name__} cannot be converted to the GPU, because it is " - "not on the CPU! This is not supposed to be possible. If you believe " - "you have found a bug, please report a minimum reproducible example to " - "https://github.com/rapidsai/cugraph/issues/new/choose" - ) - try: - Gcg = nxcg.from_networkx( - self, preserve_edge_attrs=True, preserve_node_attrs=True - ) - except Exception: - # Should we warn that the full graph can't be on GPU? - nx_cache[_CANT_CONVERT_TO_GPU] = True - return None - Gcg.graph = self.graph - cache[_CACHE_KEY] = Gcg - return Gcg - - @_cugraph.setter - def _cugraph(self, val): - if (cache := getattr(self, "__networkx_cache__", None)) is None: - # Should we warn? - return - # TODO: pay close attention to when we should clear the cache, since - # this may or may not be a mutation. - cache = cache.setdefault("backends", {}).setdefault("cugraph", {}) - if val is None: - cache.pop(_CACHE_KEY, None) - else: - self.graph = val.graph - cache[_CACHE_KEY] = val - for key in self._nx_attrs: - self.__dict__[key] = None - - @nx.Graph.name.setter - def name(self, s): - # Don't clear the cache when setting the name, since `.graph` is shared. - # There is a very small risk here for the cache to become (slightly) - # insconsistent if graphs from other backends are cached. - self.graph["name"] = s - - @classmethod - def to_networkx_class(cls): - return nx.Graph - - @classmethod - def to_cugraph_class(cls): - return nxcg.Graph - - def __init__(self, incoming_graph_data=None, **attr): - super().__init__(incoming_graph_data, **attr) - self.__networkx_cache__ = _ZeroGraphCache(self) - - def _reify_networkx(self): - if self.__dict__["_node"] is None: - # After we make this into an nx graph, we rely on the cache being correct - Gcg = self._cugraph - G = nxcg.to_networkx(Gcg) - for key in self._nx_attrs: - self.__dict__[key] = G.__dict__[key] - - -class ZeroDiGraph(nx.DiGraph, ZeroGraph): - _nx_attrs = ("_node", "_adj", "_succ", "_pred") - - name = ZeroGraph.name - _node = ZeroGraph._node - - @property - def _adj(self): - if (adj := self.__dict__["_adj"]) is None: - self._reify_networkx() - adj = self.__dict__["_adj"] - return adj - - @_adj.setter - def _adj(self, val): - _CachedPropertyResetterAdjAndSucc.__set__(None, self, val) - - @property - def _succ(self): - if (succ := self.__dict__["_succ"]) is None: - self._reify_networkx() - succ = self.__dict__["_succ"] - return succ - - @_succ.setter - def _succ(self, val): - _CachedPropertyResetterAdjAndSucc.__set__(None, self, val) - - @property - def _pred(self): - if (pred := self.__dict__["_pred"]) is None: - self._reify_networkx() - pred = self.__dict__["_pred"] - return pred - - @_pred.setter - def _pred(self, val): - _CachedPropertyResetterPred.__set__(None, self, val) - - @classmethod - def to_networkx_class(cls): - return nx.DiGraph - - @classmethod - def to_cugraph_class(cls): - return nxcg.DiGraph - - -class ZeroMultiGraph(nx.MultiGraph, ZeroGraph): - name = ZeroGraph.name - _node = ZeroGraph._node - _adj = ZeroGraph._adj - - @classmethod - def to_networkx_class(cls): - return nx.MultiGraph - - @classmethod - def to_cugraph_class(cls): - return nxcg.MultiGraph - - def __init__(self, incoming_graph_data=None, multigraph_input=None, **attr): - super().__init__(incoming_graph_data, multigraph_input, **attr) - self.__networkx_cache__ = _ZeroGraphCache(self) - - -class ZeroMultiDiGraph(nx.MultiDiGraph, ZeroMultiGraph, ZeroDiGraph): - name = ZeroGraph.name - _node = ZeroGraph._node - _adj = ZeroDiGraph._adj - _succ = ZeroDiGraph._succ - _pred = ZeroDiGraph._pred - - @classmethod - def to_networkx_class(cls): - return nx.MultiDiGraph - - @classmethod - def to_cugraph_class(cls): - return nxcg.MultiDiGraph diff --git a/python/nx-cugraph/nx_cugraph/convert.py b/python/nx-cugraph/nx_cugraph/convert.py index c122bca2c07..a872f13ac70 100644 --- a/python/nx-cugraph/nx_cugraph/convert.py +++ b/python/nx-cugraph/nx_cugraph/convert.py @@ -26,7 +26,6 @@ import nx_cugraph as nxcg from nx_cugraph import _nxver -from .classes.zero import ZeroGraph from .utils import index_dtype, networkx_algorithm from .utils.misc import _And_NotImplementedError, pairwise @@ -101,8 +100,8 @@ def from_networkx( as_directed: bool = False, name: str | None = None, graph_name: str | None = None, - zero: bool | None = False, -) -> nxcg.Graph: + use_compat_graph: bool | None = False, +) -> nxcg.Graph | nxcg.CudaGraph: """Convert a networkx graph to nx_cugraph graph; can convert all attributes. Parameters @@ -142,10 +141,16 @@ def from_networkx( The name of the algorithm when dispatched from networkx. graph_name : str, optional The name of the graph argument geing converted when dispatched from networkx. + use_compat_graph : bool or None, default False + Indicate whether to return a graph that is compatible with NetworkX graph. + For example, ``nx_cugraph.Graph`` can be used as a NetworkX graph and can + reside in host (CPU) or device (GPU) memory. The default is False, which + will return e.g. ``nx_cugraph.CudaGraph`` that only resides on device (GPU) + and is not fully compatible as a NetworkX graph. Returns ------- - nx_cugraph.Graph + nx_cugraph.Graph or nx_cugraph.CudaGraph Notes ----- @@ -173,16 +178,16 @@ def from_networkx( graph = G else: raise TypeError(f"Expected networkx.Graph; got {type(graph)}") - elif isinstance(graph, ZeroGraph): + elif isinstance(graph, nxcg.Graph): if ( - zero - or zero is None - and _nxver >= (3, 3) - and nx.config.backends.cugraph.zero + use_compat_graph + # Use compat graphs by default + or use_compat_graph is None + and (_nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs) ): return graph if graph._is_on_gpu: - return graph._cugraph + return graph._cudagraph if not graph._is_on_cpu: raise RuntimeError( f"{type(graph).__name__} cannot be converted to the GPU, because it is " @@ -203,7 +208,11 @@ def from_networkx( cache = cache.setdefault("backends", {}).setdefault("cugraph", {}) compat_key, rv = _get_from_cache(cache, cache_key) if rv is not None: - return rv + if isinstance(rv, nxcg.Graph): + # This shouldn't happen during normal use, but be extra-careful + rv = rv._cudagraph + if rv is not None: + return rv if preserve_all_attrs: preserve_edge_attrs = True @@ -229,7 +238,7 @@ def from_networkx( nx.DiGraph, nx.MultiGraph, nx.MultiDiGraph, - } or isinstance(graph, ZeroGraph): + } or isinstance(graph, nxcg.Graph): # This is a NetworkX private attribute, but is much faster to use adj = graph._adj else: @@ -519,9 +528,9 @@ def func(it, edge_attr=edge_attr, dtype=dtype): # if vals.ndim > 1: ... if graph.is_multigraph(): if graph.is_directed() or as_directed: - klass = nxcg.MultiDiGraph + klass = nxcg.CudaMultiDiGraph else: - klass = nxcg.MultiGraph + klass = nxcg.CudaMultiGraph rv = klass.from_coo( N, src_indices, @@ -533,13 +542,13 @@ def func(it, edge_attr=edge_attr, dtype=dtype): node_masks, key_to_id=key_to_id, edge_keys=edge_keys, - zero=zero, + use_compat_graph=False, ) else: if graph.is_directed() or as_directed: - klass = nxcg.DiGraph + klass = nxcg.CudaDiGraph else: - klass = nxcg.Graph + klass = nxcg.CudaGraph rv = klass.from_coo( N, src_indices, @@ -549,15 +558,22 @@ def func(it, edge_attr=edge_attr, dtype=dtype): node_values, node_masks, key_to_id=key_to_id, - zero=zero, + use_compat_graph=False, ) if preserve_graph_attrs: rv.graph.update(graph.graph) # deepcopy? - if _nxver >= (3, 4) and isinstance(graph, ZeroGraph) and cache is not None: + if _nxver >= (3, 4) and isinstance(graph, nxcg.Graph) and cache is not None: # Make sure this conversion is added to the cache, and make all of # our graphs share the same `.graph` attribute for consistency. rv.graph = graph.graph _set_to_cache(cache, cache_key, rv) + if ( + use_compat_graph + # Use compat graphs by default + or use_compat_graph is None + and (_nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs) + ): + return rv._to_compat_graph() return rv @@ -606,14 +622,16 @@ def _iter_attr_dicts( return full_dicts -def to_networkx(G: nxcg.Graph, *, sort_edges: bool = False) -> nx.Graph: +def to_networkx( + G: nxcg.Graph | nxcg.CudaGraph, *, sort_edges: bool = False +) -> nx.Graph: """Convert a nx_cugraph graph to networkx graph. All edge and node attributes and ``G.graph`` properties are converted. Parameters ---------- - G : nx_cugraph.Graph + G : nx_cugraph.Graph or nx_cugraph.CudaGraph sort_edges : bool, default False Whether to sort the edge data of the input graph by (src, dst) indices before converting. This can be useful to convert to networkx graphs @@ -628,8 +646,8 @@ def to_networkx(G: nxcg.Graph, *, sort_edges: bool = False) -> nx.Graph: -------- from_networkx : The opposite; convert networkx graph to nx_cugraph graph """ - if isinstance(G, ZeroGraph): - # ZeroGraphs are already NetworkX graphs :) + if isinstance(G, nxcg.Graph): + # These graphs are already NetworkX graphs :) return G rv = G.to_networkx_class()() id_to_key = G.id_to_key @@ -697,13 +715,13 @@ def _to_graph( edge_attr: AttrKey | None = None, edge_default: EdgeValue | None = 1, edge_dtype: Dtype | None = None, -) -> nxcg.Graph | nxcg.DiGraph: +) -> nxcg.CudaGraph | nxcg.CudaDiGraph: """Ensure that input type is a nx_cugraph graph, and convert if necessary. Directed and undirected graphs are both allowed. This is an internal utility function and may change or be removed. """ - if isinstance(G, nxcg.Graph): + if isinstance(G, nxcg.CudaGraph): return G if isinstance(G, nx.Graph): return from_networkx( @@ -718,15 +736,15 @@ def _to_directed_graph( edge_attr: AttrKey | None = None, edge_default: EdgeValue | None = 1, edge_dtype: Dtype | None = None, -) -> nxcg.DiGraph: - """Ensure that input type is a nx_cugraph DiGraph, and convert if necessary. +) -> nxcg.CudaDiGraph: + """Ensure that input type is a nx_cugraph CudaDiGraph, and convert if necessary. Undirected graphs will be converted to directed. This is an internal utility function and may change or be removed. """ - if isinstance(G, nxcg.DiGraph): + if isinstance(G, nxcg.CudaDiGraph): return G - if isinstance(G, nxcg.Graph): + if isinstance(G, nxcg.CudaGraph): return G.to_directed() if isinstance(G, nx.Graph): return from_networkx( @@ -744,13 +762,13 @@ def _to_undirected_graph( edge_attr: AttrKey | None = None, edge_default: EdgeValue | None = 1, edge_dtype: Dtype | None = None, -) -> nxcg.Graph: - """Ensure that input type is a nx_cugraph Graph, and convert if necessary. +) -> nxcg.CudaGraph: + """Ensure that input type is a nx_cugraph CudaGraph, and convert if necessary. Only undirected graphs are allowed. Directed graphs will raise ValueError. This is an internal utility function and may change or be removed. """ - if isinstance(G, nxcg.Graph): + if isinstance(G, nxcg.CudaGraph): if G.is_directed(): raise ValueError("Only undirected graphs supported; got a directed graph") return G diff --git a/python/nx-cugraph/nx_cugraph/generators/_utils.py b/python/nx-cugraph/nx_cugraph/generators/_utils.py index e38ace5b28d..bc9ab84bdad 100644 --- a/python/nx-cugraph/nx_cugraph/generators/_utils.py +++ b/python/nx-cugraph/nx_cugraph/generators/_utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -16,6 +16,7 @@ import networkx as nx import nx_cugraph as nxcg +from nx_cugraph import _nxver from ..utils import index_dtype @@ -74,7 +75,7 @@ def _common_small_graph(n, nodes, create_using, *, allow_directed=True): return G -def _create_using_class(create_using, *, default=nxcg.Graph): +def _create_using_class(create_using, *, default=nx.Graph): """Handle ``create_using`` argument and return a Graph type from nx_cugraph.""" inplace = False if create_using is None: @@ -85,16 +86,17 @@ def _create_using_class(create_using, *, default=nxcg.Graph): create_using, "is_multigraph" ): raise TypeError("create_using is not a valid graph type or instance") - elif not isinstance(create_using, nxcg.Graph): + elif not isinstance(create_using, (nxcg.Graph, nxcg.CudaGraph)): raise NotImplementedError( f"create_using with object of type {type(create_using)} is not supported " - "by the cugraph backend; only nx_cugraph.Graph objects are allowed." + "by the cugraph backend; only nx_cugraph.Graph or nx_cugraph.CudaGraph " + "objects are allowed." ) else: inplace = True G = create_using G.clear() - if not isinstance(G, nxcg.Graph): + if not isinstance(G, (nxcg.Graph, nxcg.CudaGraph)): if G.is_multigraph(): if G.is_directed(): graph_class = nxcg.MultiDiGraph @@ -104,10 +106,12 @@ def _create_using_class(create_using, *, default=nxcg.Graph): graph_class = nxcg.DiGraph else: graph_class = nxcg.Graph + if _nxver >= (3, 3) and not nx.config.backends.cugraph.use_compat_graphs: + graph_class = graph_class.to_cudagraph_class() if G.__class__ not in {nx.Graph, nx.DiGraph, nx.MultiGraph, nx.MultiDiGraph}: raise NotImplementedError( f"create_using with type {type(G)} is not supported by the cugraph " - "backend; only standard networkx or nx_cugraph Graph objects are " + "backend; only standard networkx or nx_cugraph graph objects are " "allowed (but not customized subclasses derived from them)." ) else: diff --git a/python/nx-cugraph/nx_cugraph/generators/classic.py b/python/nx-cugraph/nx_cugraph/generators/classic.py index 51e37d03285..cfcb2a3afec 100644 --- a/python/nx-cugraph/nx_cugraph/generators/classic.py +++ b/python/nx-cugraph/nx_cugraph/generators/classic.py @@ -103,9 +103,9 @@ def complete_graph(n, create_using=None): @networkx_algorithm(version_added="23.12") def complete_multipartite_graph(*subset_sizes): if not subset_sizes: - if _nxver >= (3, 3) and nx.config.backends.cugraph.zero: - return nxcg.ZeroGraph() - return nxcg.Graph() + if _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs: + return nxcg.Graph() + return nxcg.CudaGraph() try: subset_sizes = [_ensure_int(size) for size in subset_sizes] except TypeError: @@ -142,6 +142,8 @@ def complete_multipartite_graph(*subset_sizes): dst_indices, node_values={"subset": subsets_array}, id_to_key=nodes, + use_compat_graph=_nxver < (3, 3) + or nx.config.backends.cugraph.use_compat_graphs, ) diff --git a/python/nx-cugraph/nx_cugraph/generators/community.py b/python/nx-cugraph/nx_cugraph/generators/community.py index 9b0e0848de9..4e5063cc345 100644 --- a/python/nx-cugraph/nx_cugraph/generators/community.py +++ b/python/nx-cugraph/nx_cugraph/generators/community.py @@ -11,8 +11,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import cupy as cp +import networkx as nx import nx_cugraph as nxcg +from nx_cugraph import _nxver from ..utils import networkx_algorithm from ._utils import ( @@ -42,4 +44,7 @@ def caveman_graph(l, k): # noqa: E741 dst_cliques.extend(dst_clique + i * k for i in range(1, l)) src_indices = cp.hstack(src_cliques) dst_indices = cp.hstack(dst_cliques) - return nxcg.Graph.from_coo(l * k, src_indices, dst_indices) + use_compat_graph = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + return nxcg.CudaGraph.from_coo( + l * k, src_indices, dst_indices, use_compat_graph=use_compat_graph + ) diff --git a/python/nx-cugraph/nx_cugraph/generators/ego.py b/python/nx-cugraph/nx_cugraph/generators/ego.py index d4e443d1c46..9a91fa0b6c3 100644 --- a/python/nx-cugraph/nx_cugraph/generators/ego.py +++ b/python/nx-cugraph/nx_cugraph/generators/ego.py @@ -32,10 +32,10 @@ def ego_graph( ): """Weighted ego_graph with negative cycles is not yet supported. `NotImplementedError` will be raised if there are negative `distance` edge weights.""" # noqa: E501 if isinstance(G, nx.Graph): - zero = isinstance(G, nxcg.ZeroGraph) + is_compat_graph = isinstance(G, nxcg.Graph) G = nxcg.from_networkx(G, preserve_all_attrs=True) else: - zero = False + is_compat_graph = False if n not in G: if distance is None: raise nx.NodeNotFound(f"Source {n} is not in G") @@ -104,8 +104,8 @@ def ego_graph( node_ids = node_ids[node_mask] if node_ids.size == G._N: rv = G.copy() - if zero: - return rv.to_zero() + if is_compat_graph: + return rv._to_compat_graph() return rv # TODO: create renumbering helper function(s) node_ids.sort() # TODO: is this ever necessary? Keep for safety @@ -143,7 +143,7 @@ def ego_graph( "node_values": node_values, "node_masks": node_masks, "key_to_id": key_to_id, - "zero": False, + "use_compat_graph": False, } if G.is_multigraph(): if G.edge_keys is not None: @@ -154,8 +154,8 @@ def ego_graph( kwargs["edge_indices"] = G.edge_indices[edge_mask] rv = G.__class__.from_coo(**kwargs) rv.graph.update(G.graph) - if zero: - return rv.to_zero() + if is_compat_graph: + return rv._to_compat_graph() return rv diff --git a/python/nx-cugraph/nx_cugraph/generators/small.py b/python/nx-cugraph/nx_cugraph/generators/small.py index 45487571cda..d0c03cb7dd4 100644 --- a/python/nx-cugraph/nx_cugraph/generators/small.py +++ b/python/nx-cugraph/nx_cugraph/generators/small.py @@ -14,6 +14,7 @@ import networkx as nx import nx_cugraph as nxcg +from nx_cugraph import _nxver from ..utils import index_dtype, networkx_algorithm from ._utils import _IS_NX32_OR_LESS, _create_using_class @@ -449,7 +450,14 @@ def pappus_graph(): index_dtype, ) # fmt: on - return nxcg.Graph.from_coo(18, src_indices, dst_indices, name="Pappus Graph") + use_compat_graph = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + return nxcg.CudaGraph.from_coo( + 18, + src_indices, + dst_indices, + name="Pappus Graph", + use_compat_graph=use_compat_graph, + ) @networkx_algorithm(version_added="23.12") diff --git a/python/nx-cugraph/nx_cugraph/generators/social.py b/python/nx-cugraph/nx_cugraph/generators/social.py index 07e82c63fbf..09d405e7561 100644 --- a/python/nx-cugraph/nx_cugraph/generators/social.py +++ b/python/nx-cugraph/nx_cugraph/generators/social.py @@ -11,9 +11,11 @@ # See the License for the specific language governing permissions and # limitations under the License. import cupy as cp +import networkx as nx import numpy as np import nx_cugraph as nxcg +from nx_cugraph import _nxver from ..utils import index_dtype, networkx_algorithm @@ -77,7 +79,8 @@ def davis_southern_women_graph(): "E13", "E14", ] # fmt: on - return nxcg.Graph.from_coo( + use_compat_graph = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + return nxcg.CudaGraph.from_coo( 32, src_indices, dst_indices, @@ -85,6 +88,7 @@ def davis_southern_women_graph(): id_to_key=women + events, top=women, bottom=events, + use_compat_graph=use_compat_graph, ) @@ -111,7 +115,14 @@ def florentine_families_graph(): "Salviati", "Strozzi", "Tornabuoni" ] # fmt: on - return nxcg.Graph.from_coo(15, src_indices, dst_indices, id_to_key=nodes) + use_compat_graph = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + return nxcg.CudaGraph.from_coo( + 15, + src_indices, + dst_indices, + id_to_key=nodes, + use_compat_graph=use_compat_graph, + ) @networkx_algorithm(version_added="23.12") @@ -165,13 +176,15 @@ def karate_club_graph(): "Officer", "Officer", "Officer", "Officer", "Officer", "Officer", ]) # fmt: on - return nxcg.Graph.from_coo( + use_compat_graph = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + return nxcg.CudaGraph.from_coo( 34, src_indices, dst_indices, edge_values={"weight": weights}, node_values={"club": clubs}, name="Zachary's Karate Club", + use_compat_graph=use_compat_graph, ) @@ -289,6 +302,12 @@ def les_miserables_graph(): "Zephine", ] # fmt: on - return nxcg.Graph.from_coo( - 77, src_indices, dst_indices, edge_values={"weight": weights}, id_to_key=nodes + use_compat_graph = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + return nxcg.CudaGraph.from_coo( + 77, + src_indices, + dst_indices, + edge_values={"weight": weights}, + id_to_key=nodes, + use_compat_graph=use_compat_graph, ) diff --git a/python/nx-cugraph/nx_cugraph/interface.py b/python/nx-cugraph/nx_cugraph/interface.py index 8910e463904..1a3d08409a2 100644 --- a/python/nx-cugraph/nx_cugraph/interface.py +++ b/python/nx-cugraph/nx_cugraph/interface.py @@ -37,14 +37,15 @@ def convert_from_nx(graph, *args, edge_attrs=None, weight=None, **kwargs): graph, *args, edge_attrs=edge_attrs, - zero=_nxver >= (3, 3) and nx.config.backends.cugraph.zero, + use_compat_graph=_nxver < (3, 3) + or nx.config.backends.cugraph.use_compat_graphs, **kwargs, ) @staticmethod def convert_to_nx(obj, *, name: str | None = None): - if isinstance(obj, nxcg.Graph): - # Observe that this does not try to convert ZeroGraph! + if isinstance(obj, nxcg.CudaGraph): + # Observe that this does not try to convert Graph! return nxcg.to_networkx(obj) return obj @@ -70,8 +71,10 @@ def key(testpath): return (testname, frozenset({classname, filename})) return (testname, frozenset({filename})) - zero = _nxver >= (3, 3) and nx.config.backends.cugraph.zero - fallback = zero or nx.utils.backends._dispatchable._fallback_to_nx + use_compat_graph = ( + _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + ) + fallback = use_compat_graph or nx.utils.backends._dispatchable._fallback_to_nx # Reasons for xfailing # For nx version <= 3.1 @@ -89,7 +92,7 @@ def key(testpath): # This one is tricky b/c we don't raise; all dtypes are treated as str "mixed dtypes (str, int, float) for single node property not supported" ) - # These shouldn't fail if using ZeroGraph or falling back to networkx + # These shouldn't fail if using Graph or falling back to networkx no_string_dtype = "string edge values not currently supported" no_object_dtype_for_edges = ( "Edges don't support object dtype (lists, strings, etc.)" diff --git a/python/nx-cugraph/nx_cugraph/relabel.py b/python/nx-cugraph/nx_cugraph/relabel.py index fe18183dd45..e38e18c779e 100644 --- a/python/nx-cugraph/nx_cugraph/relabel.py +++ b/python/nx-cugraph/nx_cugraph/relabel.py @@ -29,16 +29,17 @@ @networkx_algorithm(version_added="24.08") def relabel_nodes(G, mapping, copy=True): + G_orig = G if isinstance(G, nx.Graph): - zero = isinstance(G, nxcg.ZeroGraph) - if not copy and not zero: + is_compat_graph = isinstance(G, nxcg.Graph) + if not copy and not is_compat_graph: raise RuntimeError( "Using `copy=False` is invalid when using a NetworkX graph " "as input to `nx_cugraph.relabel_nodes`" ) G = nxcg.from_networkx(G, preserve_all_attrs=True) else: - zero = False + is_compat_graph = False it = range(G._N) if G.key_to_id is None else G.id_to_key if callable(mapping): @@ -229,17 +230,13 @@ def relabel_nodes(G, mapping, copy=True): node_masks=node_masks, id_to_key=newid_to_key, key_to_id=key_to_newid, - zero=False, # XXX TODO: figure out `create_using=` w/ zero graphs + use_compat_graph=is_compat_graph, **extra_kwargs, ) rv.graph.update(G.graph) if not copy: - G._become(rv) - if zero: - return rv.to_zero() # TODO: update original graph - return G - if zero: - return rv.to_zero() + G_orig._become(rv) + return G_orig return rv @@ -250,10 +247,10 @@ def convert_node_labels_to_integers( if ordering not in {"default", "sorted", "increasing degree", "decreasing degree"}: raise nx.NetworkXError(f"Unknown node ordering: {ordering}") if isinstance(G, nx.Graph): - zero = isinstance(G, nxcg.ZeroGraph) + is_compat_graph = isinstance(G, nxcg.Graph) G = nxcg.from_networkx(G, preserve_all_attrs=True) else: - zero = False + is_compat_graph = False G = G.copy() if label_attribute is not None: prev_vals = G.id_to_key @@ -291,6 +288,6 @@ def convert_node_labels_to_integers( key_to_id = G.key_to_id G.key_to_id = {i: key_to_id[n] for i, (d, n) in enumerate(pairs, first_label)} G._id_to_key = id_to_key - if zero: - return G.to_zero() + if is_compat_graph: + return G._to_compat_graph() return G diff --git a/python/nx-cugraph/nx_cugraph/tests/test_classes.py b/python/nx-cugraph/nx_cugraph/tests/test_classes.py new file mode 100644 index 00000000000..0ac238b3558 --- /dev/null +++ b/python/nx-cugraph/nx_cugraph/tests/test_classes.py @@ -0,0 +1,77 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import nx_cugraph as nxcg + + +def test_class_to_class(): + """Basic sanity checks to ensure metadata relating graph classes are accurate.""" + for prefix in ["", "Cuda"]: + for suffix in ["Graph", "DiGraph", "MultiGraph", "MultiDiGraph"]: + cls_name = f"{prefix}{suffix}" + cls = getattr(nxcg, cls_name) + assert cls.__name__ == cls_name + G = cls() + assert cls is G.__class__ + # cudagraph + val = cls.to_cudagraph_class() + val2 = G.to_cudagraph_class() + assert val is val2 + assert val.__name__ == f"Cuda{suffix}" + assert val.__module__.startswith("nx_cugraph") + assert cls.is_directed() == G.is_directed() == val.is_directed() + assert cls.is_multigraph() == G.is_multigraph() == val.is_multigraph() + # networkx + val = cls.to_networkx_class() + val2 = G.to_networkx_class() + assert val is val2 + assert val.__name__ == suffix + assert val.__module__.startswith("networkx") + val = val() + assert cls.is_directed() == G.is_directed() == val.is_directed() + assert cls.is_multigraph() == G.is_multigraph() == val.is_multigraph() + # directed + val = cls.to_directed_class() + val2 = G.to_directed_class() + assert val is val2 + assert val.__module__.startswith("nx_cugraph") + assert val.is_directed() + assert cls.is_multigraph() == G.is_multigraph() == val.is_multigraph() + if "Di" in suffix: + assert val is cls + else: + assert "Di" in val.__name__ + assert prefix in val.__name__ + assert cls.to_undirected_class() is cls + # undirected + val = cls.to_undirected_class() + val2 = G.to_undirected_class() + assert val is val2 + assert val.__module__.startswith("nx_cugraph") + assert not val.is_directed() + assert cls.is_multigraph() == G.is_multigraph() == val.is_multigraph() + if "Di" not in suffix: + assert val is cls + else: + assert "Di" not in val.__name__ + assert prefix in val.__name__ + assert cls.to_directed_class() is cls + # "zero" + if prefix == "Cuda": + val = cls._to_compat_graph_class() + val2 = G._to_compat_graph_class() + assert val is val2 + assert val.__name__ == suffix + assert val.__module__.startswith("nx_cugraph") + assert val.to_cudagraph_class() is cls + assert cls.is_directed() == G.is_directed() == val.is_directed() + assert cls.is_multigraph() == G.is_multigraph() == val.is_multigraph() diff --git a/python/nx-cugraph/nx_cugraph/tests/test_ego_graph.py b/python/nx-cugraph/nx_cugraph/tests/test_ego_graph.py index c4e5c9cfbb7..0697a744e85 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_ego_graph.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_ego_graph.py @@ -46,7 +46,12 @@ def test_ego_graph_cycle_graph( kwargs = {"radius": radius, "center": center, "undirected": undirected} Hnx = nx.ego_graph(Gnx, n, **kwargs) Hcg = nx.ego_graph(Gnx, n, **kwargs, backend="cugraph") + use_compat_graphs = _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs + assert_graphs_equal(Hnx, Hcg._cudagraph if use_compat_graphs else Hcg) + Hcg = nx.ego_graph(Gcg, n, **kwargs) assert_graphs_equal(Hnx, Hcg) + Hcg = nx.ego_graph(Gcg._to_compat_graph(), n, **kwargs) + assert_graphs_equal(Hnx, Hcg._cudagraph) with pytest.raises(nx.NodeNotFound, match="not in G"): nx.ego_graph(Gnx, -1, **kwargs) with pytest.raises(nx.NodeNotFound, match="not in G"): @@ -63,15 +68,31 @@ def test_ego_graph_cycle_graph( if is_nx32: # `should_run` was added in nx 3.3 match = "Weighted ego_graph with undirected=True not implemented" + elif _nxver >= (3, 4): + match = "not implemented by 'cugraph'" else: match = "not implemented by cugraph" - with pytest.raises(RuntimeError, match=match): + with pytest.raises( + RuntimeError if _nxver < (3, 4) else NotImplementedError, match=match + ): nx.ego_graph(Gnx, n, **kwargs, backend="cugraph") with pytest.raises(NotImplementedError, match="ego_graph"): - nx.ego_graph(Gcg, n, **kwargs) + nx.ego_graph(Gcg, n, **kwargs, backend="cugraph") + if _nxver < (3, 4): + with pytest.raises(NotImplementedError, match="ego_graph"): + nx.ego_graph(Gcg, n, **kwargs) + else: + # This is an interesting case. `nxcg.ego_graph` is not implemented for + # these arguments, so it falls back to networkx. Hence, as it is currently + # implemented, the input graph is `nxcg.CudaGraph`, but the output graph + # is `nx.Graph`. Should networkx convert back to "cugraph" backend? + # TODO: make fallback to networkx configurable. + H2cg = nx.ego_graph(Gcg, n, **kwargs) + assert type(H2nx) is type(H2cg) + assert_graphs_equal(H2nx, nxcg.from_networkx(H2cg, preserve_all_attrs=True)) else: H2cg = nx.ego_graph(Gnx, n, **kwargs, backend="cugraph") - assert_graphs_equal(H2nx, H2cg) + assert_graphs_equal(H2nx, H2cg._cudagraph if use_compat_graphs else H2cg) with pytest.raises(nx.NodeNotFound, match="not found in graph"): nx.ego_graph(Gnx, -1, **kwargs) with pytest.raises(nx.NodeNotFound, match="not found in graph"): diff --git a/python/nx-cugraph/nx_cugraph/tests/test_generators.py b/python/nx-cugraph/nx_cugraph/tests/test_generators.py index 2fc97660047..5c405f1c93b 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_generators.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_generators.py @@ -26,9 +26,11 @@ def compare(name, create_using, *args, is_vanilla=False): exc1 = exc2 = None func = getattr(nx, name) - if isinstance(create_using, nxcg.Graph): + if isinstance(create_using, nxcg.CudaGraph): nx_create_using = nxcg.to_networkx(create_using) - elif isinstance(create_using, type) and issubclass(create_using, nxcg.Graph): + elif isinstance(create_using, type) and issubclass( + create_using, (nxcg.Graph, nxcg.CudaGraph) + ): nx_create_using = create_using.to_networkx_class() elif isinstance(create_using, nx.Graph): nx_create_using = create_using.copy() @@ -58,8 +60,27 @@ def compare(name, create_using, *args, is_vanilla=False): exc2 = exc if exc1 is not None or exc2 is not None: assert type(exc1) is type(exc2) + return + if isinstance(Gcg, nxcg.Graph): + # If the graph is empty, it may be on host, otherwise it should be on device + if len(G): + assert Gcg._is_on_gpu + assert not Gcg._is_on_cpu + assert_graphs_equal(G, Gcg._cudagraph) else: assert_graphs_equal(G, Gcg) + # Ensure the output type is correct + if is_vanilla: + if _nxver < (3, 3) or nx.config.backends.cugraph.use_compat_graphs: + assert isinstance(Gcg, nxcg.Graph) + else: + assert isinstance(Gcg, nxcg.CudaGraph) + elif isinstance(create_using, type) and issubclass( + create_using, (nxcg.Graph, nxcg.CudaGraph) + ): + assert type(Gcg) is create_using + elif isinstance(create_using, (nxcg.Graph, nxcg.CudaGraph)): + assert type(Gcg) is type(create_using) N = list(range(-1, 5)) @@ -73,6 +94,10 @@ def compare(name, create_using, *args, is_vanilla=False): nxcg.DiGraph, nxcg.MultiGraph, nxcg.MultiDiGraph, + nxcg.CudaGraph, + nxcg.CudaDiGraph, + nxcg.CudaMultiGraph, + nxcg.CudaMultiDiGraph, # These raise NotImplementedError # nx.Graph(), # nx.DiGraph(), @@ -82,6 +107,10 @@ def compare(name, create_using, *args, is_vanilla=False): nxcg.DiGraph(), nxcg.MultiGraph(), nxcg.MultiDiGraph(), + nxcg.CudaGraph(), + nxcg.CudaDiGraph(), + nxcg.CudaMultiGraph(), + nxcg.CudaMultiDiGraph(), None, object, # Bad input 7, # Bad input @@ -155,7 +184,7 @@ def compare(name, create_using, *args, is_vanilla=False): @pytest.mark.parametrize("create_using", COMPLETE_CREATE_USING) def test_generator_noarg(name, create_using): print(name, create_using, type(create_using)) - if isinstance(create_using, nxcg.Graph) and name in { + if isinstance(create_using, nxcg.CudaGraph) and name in { # fmt: off "bull_graph", "chvatal_graph", "cubical_graph", "diamond_graph", "house_graph", "house_x_graph", "icosahedral_graph", "krackhardt_kite_graph", diff --git a/python/nx-cugraph/nx_cugraph/tests/test_graph_methods.py b/python/nx-cugraph/nx_cugraph/tests/test_graph_methods.py index 3120995a2b2..40a361b1084 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_graph_methods.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_graph_methods.py @@ -47,7 +47,7 @@ def _create_Gs(): @pytest.mark.parametrize("Gnx", _create_Gs()) @pytest.mark.parametrize("reciprocal", [False, True]) def test_to_undirected_directed(Gnx, reciprocal): - Gcg = nxcg.DiGraph(Gnx) + Gcg = nxcg.CudaDiGraph(Gnx) assert_graphs_equal(Gnx, Gcg) Hnx1 = Gnx.to_undirected(reciprocal=reciprocal) Hcg1 = Gcg.to_undirected(reciprocal=reciprocal) @@ -62,6 +62,6 @@ def test_multidigraph_to_undirected(): Gnx.add_edge(0, 1) Gnx.add_edge(0, 1) Gnx.add_edge(1, 0) - Gcg = nxcg.MultiDiGraph(Gnx) + Gcg = nxcg.CudaMultiDiGraph(Gnx) with pytest.raises(NotImplementedError): Gcg.to_undirected() diff --git a/python/nx-cugraph/nx_cugraph/tests/test_multigraph.py b/python/nx-cugraph/nx_cugraph/tests/test_multigraph.py index a8f189a4745..9208eea09f2 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_multigraph.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_multigraph.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -26,7 +26,7 @@ def test_get_edge_data(test_nxcugraph): G.add_edge(0, 3) G.add_edge(0, 3) if test_nxcugraph: - G = nxcg.MultiGraph(G) + G = nxcg.CudaMultiGraph(G) default = object() assert G.get_edge_data(0, 0, default=default) is default assert G.get_edge_data("a", "b", default=default) is default @@ -60,7 +60,7 @@ def test_get_edge_data(test_nxcugraph): G = nx.MultiGraph() G.add_edge(0, 1) if test_nxcugraph: - G = nxcg.MultiGraph(G) + G = nxcg.CudaMultiGraph(G) assert G.get_edge_data(0, 1, default=default) == {0: {}} assert G.get_edge_data(0, 1, 0, default=default) == {} assert G.get_edge_data(0, 1, 1, default=default) is default diff --git a/python/nx-cugraph/nx_cugraph/tests/testing_utils.py b/python/nx-cugraph/nx_cugraph/tests/testing_utils.py index 529a96efd81..50836acf55f 100644 --- a/python/nx-cugraph/nx_cugraph/tests/testing_utils.py +++ b/python/nx-cugraph/nx_cugraph/tests/testing_utils.py @@ -17,7 +17,7 @@ def assert_graphs_equal(Gnx, Gcg): assert isinstance(Gnx, nx.Graph) - assert isinstance(Gcg, nxcg.Graph) + assert isinstance(Gcg, nxcg.CudaGraph) assert (a := Gnx.number_of_nodes()) == (b := Gcg.number_of_nodes()), (a, b) assert (a := Gnx.number_of_edges()) == (b := Gcg.number_of_edges()), (a, b) assert (a := Gnx.is_directed()) == (b := Gcg.is_directed()), (a, b) diff --git a/python/nx-cugraph/nx_cugraph/utils/misc.py b/python/nx-cugraph/nx_cugraph/utils/misc.py index 4831fe93894..01c25dd5983 100644 --- a/python/nx-cugraph/nx_cugraph/utils/misc.py +++ b/python/nx-cugraph/nx_cugraph/utils/misc.py @@ -194,7 +194,7 @@ def _get_int_dtype( def _get_float_dtype( - dtype: Dtype, *, graph: nxcg.Graph | None = None, weight: EdgeKey | None = None + dtype: Dtype, *, graph: nxcg.CudaGraph | None = None, weight: EdgeKey | None = None ): """Promote dtype to float32 or float64 as appropriate.""" if dtype is None: diff --git a/python/nx-cugraph/run_nx_tests.sh b/python/nx-cugraph/run_nx_tests.sh index b67343c7d3c..5fb173cf939 100755 --- a/python/nx-cugraph/run_nx_tests.sh +++ b/python/nx-cugraph/run_nx_tests.sh @@ -18,9 +18,9 @@ # testing takes longer. Without it, tests will xfail when encountering a # function that we don't implement. # -# NX_CUGRAPH_ZERO, {"True", "False"}, default is "True" -# Whether to use `nxcg.ZeroGraph` as the nx_cugraph backend graph. -# A ZeroGraph should be a compatible NetworkX graph, so fewer tests should fail. +# NX_CUGRAPH_USE_COMPAT_GRAPHS, {"True", "False"}, default is "True" +# Whether to use `nxcg.Graph` as the nx_cugraph backend graph. +# A Graph should be a compatible NetworkX graph, so fewer tests should fail. # # Coverage of `nx_cugraph.algorithms` is reported and is a good sanity check # that algorithms run. From a8502cb6258e22cb8437c1a14b5b4bfff8121266 Mon Sep 17 00:00:00 2001 From: Erik Welch Date: Mon, 23 Sep 2024 04:59:49 -0700 Subject: [PATCH 8/8] Update and comment based on feedback --- python/nx-cugraph/lint.yaml | 4 ++-- python/nx-cugraph/nx_cugraph/classes/graph.py | 14 +++++++++++-- .../nx_cugraph/tests/test_pagerank.py | 20 +++++++++++-------- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/python/nx-cugraph/lint.yaml b/python/nx-cugraph/lint.yaml index f723eabb3e8..dab2ea70ef1 100644 --- a/python/nx-cugraph/lint.yaml +++ b/python/nx-cugraph/lint.yaml @@ -50,7 +50,7 @@ repos: - id: black # - id: black-jupyter - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.4 + rev: v0.6.7 hooks: - id: ruff args: [--fix-only, --show-fixes] # --unsafe-fixes] @@ -77,7 +77,7 @@ repos: additional_dependencies: [tomli] files: ^(nx_cugraph|docs)/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.4 + rev: v0.6.7 hooks: - id: ruff - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/python/nx-cugraph/nx_cugraph/classes/graph.py b/python/nx-cugraph/nx_cugraph/classes/graph.py index 406dfa1a0fc..cfe1e1c87e9 100644 --- a/python/nx-cugraph/nx_cugraph/classes/graph.py +++ b/python/nx-cugraph/nx_cugraph/classes/graph.py @@ -49,10 +49,20 @@ networkx_api = nxcg.utils.decorators.networkx_class(nx.Graph) +# The "everything" cache key is an internal implementation detail of NetworkX +# that may change between releases. if _nxver < (3, 4): - _CACHE_KEY = (True, True, True) + _CACHE_KEY = ( + True, # Include all edge values + True, # Include all node values + True, # Include `.graph` attributes + ) else: - _CACHE_KEY = (True, True) + _CACHE_KEY = ( + True, # Include all edge values + True, # Include all node values + # `.graph` attributes are always included now + ) # Use to indicate when a full conversion to GPU failed so we don't try again. _CANT_CONVERT_TO_GPU = "_CANT_CONVERT_TO_GPU" diff --git a/python/nx-cugraph/nx_cugraph/tests/test_pagerank.py b/python/nx-cugraph/nx_cugraph/tests/test_pagerank.py index 0b437df2d2f..252f9e6bbb8 100644 --- a/python/nx-cugraph/nx_cugraph/tests/test_pagerank.py +++ b/python/nx-cugraph/nx_cugraph/tests/test_pagerank.py @@ -12,19 +12,23 @@ # limitations under the License. import networkx as nx import pandas as pd -from pytest import approx +import pytest def test_pagerank_multigraph(): """ - Ensures correct differences between pagerank results for Graphs - vs. MultiGraphs generated using from_pandas_edgelist() + Ensures correct pagerank for Graphs and MultiGraphs when using from_pandas_edgelist. + + PageRank for MultiGraph should give different result compared to Graph; when using + a Graph, the duplicate edges should be dropped. """ - df = pd.DataFrame({"source": [0, 1, 1, 1, 1, 1, 1, 2], - "target": [1, 2, 2, 2, 2, 2, 2, 3]}) + df = pd.DataFrame( + {"source": [0, 1, 1, 1, 1, 1, 1, 2], "target": [1, 2, 2, 2, 2, 2, 2, 3]} + ) expected_pr_for_G = nx.pagerank(nx.from_pandas_edgelist(df)) expected_pr_for_MultiG = nx.pagerank( - nx.from_pandas_edgelist(df, create_using=nx.MultiGraph)) + nx.from_pandas_edgelist(df, create_using=nx.MultiGraph) + ) G = nx.from_pandas_edgelist(df, backend="cugraph") actual_pr_for_G = nx.pagerank(G, backend="cugraph") @@ -32,5 +36,5 @@ def test_pagerank_multigraph(): MultiG = nx.from_pandas_edgelist(df, create_using=nx.MultiGraph, backend="cugraph") actual_pr_for_MultiG = nx.pagerank(MultiG, backend="cugraph") - assert actual_pr_for_G == approx(expected_pr_for_G) - assert actual_pr_for_MultiG == approx(expected_pr_for_MultiG) + assert actual_pr_for_G == pytest.approx(expected_pr_for_G) + assert actual_pr_for_MultiG == pytest.approx(expected_pr_for_MultiG)