Fix mypy type errors and add mypy to .drone.yml

Fixes #1311

Add `mypy` to .drone.yml and fix type errors that come up.

Not type checking examples or tests.

Other changes:

- fix return value for `Graph.serialize` (refs #1394)
- remove default value for
  `rdflib.plugins.sparql.algebra.translateAlgebra` (refs #1322)
- add .dockerignore to reduce context size and make docker quicker
  to run.
- add .flake8 config to ignore line length as black is managing
  formatting.
- add mypy to docker-compose, makefile and tox.ini
- fix the way csv2rdf is invoked to ensure that the right code gets
  executed.

.dockerignore

@@ -0,0 +1,4 @@
+.tox
+.venv
+.mypy_cache
+.git

.drone.yml

@@ -26,8 +26,9 @@ steps:
   - pip install --default-timeout 60 -r requirements.dev.txt
   - pip install --default-timeout 60 coveralls && export HAS_COVERALLS=1
   - python setup.py install
-  - black --config black.toml --check ./rdflib | true
+  - black --config black.toml --check ./rdflib || true
   - flake8 --exit-zero rdflib
+  - mypy --show-error-context --show-error-codes rdflib
   - PYTHONWARNINGS=default nosetests --with-timer --timer-top-n 42 --with-coverage --cover-tests --cover-package=rdflib
   - coverage report --skip-covered
   - coveralls

.flake8

@@ -0,0 +1,6 @@
+# https://flake8.pycqa.org/en/latest/user/configuration.html
+[flake8]
+extend-ignore =
+    # E501: line too long
+    # Disabled so that black can control line length.
+    E501,

Makefile

@@ -2,6 +2,7 @@ tests:
 	docker-compose -f docker-compose.tests.yml up test-runner
 	docker-compose -f docker-compose.tests.yml down
 
+.PHONY: build
 build:
 	docker-compose -f docker-compose.tests.yml build
 
@@ -14,3 +15,6 @@ reformat:
 
 check-format:
 	black --config ./black.toml --check .
+
+check-types:
+	docker-compose -f docker-compose.tests.yml up check-types

docker-compose.tests.yml

@@ -13,4 +13,12 @@ services:
       dockerfile: test/Dockerfile
     volumes:
       - .:/rdflib
-    command: ["/rdflib/run_tests_with_coverage_report.sh"]
+    command: ["/rdflib/run_tests_with_coverage_report.sh"]
+
+  check-types:
+    build:
+      context: .
+      dockerfile: test/Dockerfile
+    volumes:
+      - .:/rdflib
+    command: ["python", "-m", "mypy", "--show-error-context", "--show-error-codes" ,"/rdflib/rdflib"]

docs/developers.rst

@@ -15,6 +15,9 @@ Code should be formatted using `black <https://github.com/psf/black>`_.
 While not yet mandatory, it will be required in the future  (6.0.0+).1
 Use Black v21.6b1, with the black.toml config file provided.
 
+Code should also pass `flake8 <https://github.com/psf/black>`_ linting
+and `mypy <http://mypy-lang.org/>`_ type checking.
+
 Any new functionality being added to RDFLib should have doc tests and
 unit tests. Tests should be added for any functionality being changed
 that currently does not have any doc tests or unit tests. And all the
@@ -28,7 +31,7 @@ Running tests
 -------------
 Run tests with `nose <https://nose.readthedocs.org/en/latest/>`_:
 
-.. code-block: bash
+.. code-block:: bash
 
    $ pip install nose
    $ python run_tests.py
@@ -42,10 +45,31 @@ Specific tests can either be run by module name or file name. For example::
   $ python run_tests.py --tests rdflib.graph
   $ python run_tests.py --tests test/test_graph.py
 
+Running static checks
+---------------------
+
+Check formatting with `black <https://github.com/psf/black>`_:
+
+.. code-block:: bash
+
+    python -m black --config black.toml --check ./rdflib
+
+Check style and conventions with `flake8 <https://github.com/psf/black>`_:
+
+.. code-block:: bash
+
+    python -m flake8 rdflib
+
+Check types with `mypy <http://mypy-lang.org/>`_:
+
+.. code-block:: bash
+
+    python -m mypy --show-error-context --show-error-codes rdflib
+
 Writing documentation
 ---------------------
 
-We use sphinx for generating HTML docs, see :ref:`docs`
+We use sphinx for generating HTML docs, see :ref:`docs`.
 
 Continuous Integration
 ----------------------

rdflib/compare.py

@@ -454,15 +454,15 @@ def _traces(
             experimental = self._experimental_path(coloring_copy)
             experimental_score = set([c.key() for c in experimental])
             if last_coloring:
-                generator = self._create_generator(
+                generator = self._create_generator(  # type: ignore[unreachable]
                     [last_coloring, experimental], generator
                 )
             last_coloring = experimental
-            if best_score is None or best_score < color_score:
+            if best_score is None or best_score < color_score:  # type: ignore[unreachable]
                 best = [refined_coloring]
                 best_score = color_score
                 best_experimental_score = experimental_score
-            elif best_score > color_score:
+            elif best_score > color_score:  # type: ignore[unreachable]
                 # prune this branch.
                 if stats is not None:
                     stats["prunings"] += 1
@@ -480,7 +480,7 @@ def _traces(
                 d = [depth[0]]
                 new_color = self._traces(coloring, stats=stats, depth=d)
                 color_score = tuple([c.key() for c in refined_coloring])
-                if best_score is None or color_score > best_score:
+                if best_score is None or color_score > best_score:  # type: ignore[unreachable]
                     discrete = [new_color]
                     best_score = color_score
                     best_depth = d[0]

rdflib/extras/infixowl.py

@@ -331,9 +331,6 @@ def castToQName(x):
             if isinstance(thing, BNode):
                 return thing.n3()
             return "<" + thing + ">"
-            logger.debug(list(store.objects(subject=thing, predicate=RDF.type)))
-            raise
-            return "[]"  # +thing._id.encode('utf-8')+'</em>'
         label = first(Class(thing, graph=store).label)
         if label:
             return label

rdflib/graph.py

@@ -1057,34 +1057,34 @@ def serialize(
     @overload
     def serialize(
         self,
-        destination: Union[str, BufferedIOBase],
+        destination: Union[str, BufferedIOBase, pathlib.PurePath],
         format: str = ...,
         base: Optional[str] = ...,
         encoding: Optional[str] = ...,
         **args,
-    ) -> None:
+    ) -> "Graph":
         ...
 
     # fallback
     @overload
     def serialize(
         self,
-        destination: Union[str, BufferedIOBase, None] = None,
+        destination: Union[str, BufferedIOBase, pathlib.PurePath, None] = None,
         format: str = "turtle",
         base: Optional[str] = None,
         encoding: Optional[str] = None,
         **args,
-    ) -> Optional[Union[bytes, str]]:
+    ) -> Union[bytes, str, "Graph"]:
         ...
 
     def serialize(
         self,
-        destination: Union[str, BufferedIOBase, None] = None,
+        destination: Union[str, BufferedIOBase, pathlib.PurePath, None] = None,
         format: str = "turtle",
         base: Optional[str] = None,
         encoding: Optional[str] = None,
         **args,
-    ) -> Optional[Union[bytes, str]]:
+    ) -> Union[bytes, str, "Graph"]:
         """Serialize the Graph to destination
 
         If destination is None serialize method returns the serialization as
@@ -1123,10 +1123,10 @@ def serialize(
                 location = cast(str, destination)
             scheme, netloc, path, params, _query, fragment = urlparse(location)
             if netloc != "":
-                print(
+                logger.warning(
                     "WARNING: not saving as location" + "is not a local file reference"
                 )
-                return None
+                return self
             fd, name = tempfile.mkstemp()
             stream = os.fdopen(fd, "wb")
             serializer.serialize(stream, base=base, encoding=encoding, **args)
@@ -1967,7 +1967,6 @@ def __iter__(self) -> Generator[DatasetQuad, None, None]:
         return self.quads((None, None, None, None))
 
 
-
 class QuotedGraph(Graph):
     """
     Quoted Graphs are intended to implement Notation 3 formulae. They are

rdflib/plugins/parsers/jsonld.py

@@ -35,7 +35,8 @@
 
 import warnings
 from rdflib.graph import ConjunctiveGraph
-from rdflib.parser import Parser, URLInputSource
+from rdflib.parser import URLInputSource
+import rdflib.parser
 from rdflib.namespace import RDF, XSD
 from rdflib.term import URIRef, BNode, Literal
 
@@ -78,12 +79,12 @@
     pass
 
 
-TYPE_TERM = Term(str(RDF.type), TYPE, VOCAB)
+TYPE_TERM = Term(str(RDF.type), TYPE, VOCAB)  # type: ignore[call-arg]
 
 ALLOW_LISTS_OF_LISTS = True  # NOTE: Not allowed in JSON-LD 1.0
 
 
-class JsonLDParser(Parser):
+class JsonLDParser(rdflib.parser.Parser):
     def __init__(self):
         super(JsonLDParser, self).__init__()
 

rdflib/plugins/parsers/notation3.py

@@ -1168,7 +1168,7 @@ def uri_ref2(self, argstr, i, res):
             pfx, ln = qn[0]
             if pfx is None:
                 assert 0, "not used?"
-                ns = self._baseURI + ADDED_HASH
+                ns = self._baseURI + ADDED_HASH  # type: ignore[unreachable]
             else:
                 try:
                     ns = self._bindings[pfx]

rdflib/plugins/parsers/rdfxml.py

@@ -2,7 +2,7 @@
 An RDF/XML parser for RDFLib
 """
 
-from xml.sax import make_parser, handler
+from xml.sax import make_parser, handler, xmlreader
 from xml.sax.handler import ErrorHandler
 from xml.sax.saxutils import quoteattr, escape
 
@@ -266,7 +266,7 @@ def convert(self, name, qname, attrs):
                 pass
             elif att in UNQUALIFIED:
                 # if not RDFNS[att] in atts:
-                atts[RDFNS[att]] = v
+                atts[RDFNS[att]] = v  # type: ignore[misc]
             else:
                 atts[URIRef(att)] = v
         return name, atts
@@ -474,7 +474,7 @@ def property_element_start(self, name, qname, attrs):
                     o = URIRef(atts[att])
                 else:
                     if datatype is not None:
-                        language = None
+                        language = None  # type: ignore[unreachable]
                     o = Literal(atts[att], language, datatype)
 
                 if object is None:
@@ -575,12 +575,12 @@ def literal_element_end(self, name, qname):
         self.parent.object += self.current.object + end
 
 
-def create_parser(target, store):
+def create_parser(target, store) -> xmlreader.XMLReader:
     parser = make_parser()
     try:
         # Workaround for bug in expatreader.py. Needed when
         # expatreader is trying to guess a prefix.
-        parser.start_namespace_decl("xml", "http://www.w3.org/XML/1998/namespace")
+        parser.start_namespace_decl("xml", "http://www.w3.org/XML/1998/namespace")  # type: ignore[attr-defined]
     except AttributeError:
         pass  # Not present in Jython (at least)
     parser.setFeature(handler.feature_namespaces, 1)

rdflib/plugins/shared/jsonld/context.py

@@ -544,4 +544,4 @@ def _get_source_id(self, source, key):
     "Term",
     "id, name, type, container, index, language, reverse, context," "prefix, protected",
 )
-Term.__new__.__defaults__ = (UNDEF, UNDEF, UNDEF, UNDEF, False, UNDEF, False, False)
+Term.__new__.__defaults__ = (UNDEF, UNDEF, UNDEF, UNDEF, False, UNDEF, False, False)  # type: ignore[attr-defined]

rdflib/plugins/shared/jsonld/util.py

@@ -1,12 +1,16 @@
 # -*- coding: utf-8 -*-
 # https://github.com/RDFLib/rdflib-jsonld/blob/feature/json-ld-1.1/rdflib_jsonld/util.py
+import typing as t
 
-try:
+if t.TYPE_CHECKING:
     import json
+else:
+    try:
+        import json
 
-    assert json  # workaround for pyflakes issue #13
-except ImportError:
-    import simplejson as json
+        assert json  # workaround for pyflakes issue #13
+    except ImportError:
+        import simplejson as json
 
 from os import sep
 from os.path import normpath

rdflib/plugins/sparql/algebra.py

@@ -801,7 +801,7 @@ class ExpressionNotCoveredException(Exception):
     pass
 
 
-def translateAlgebra(query_algebra: Query = None):
+def translateAlgebra(query_algebra: Query):
     """
 
     :param query_algebra: An algebra returned by the function call algebra.translateQuery(parse_tree).

rdflib/plugins/sparql/sparql.py

@@ -78,7 +78,7 @@ def __len__(self) -> int:
         while d is not None:
             i += len(d._d)
             d = d.outer
-        return i
+        return i  # type: ignore[unreachable]
 
     def __iter__(self):
         d = self

rdflib/store.py

@@ -364,8 +364,6 @@ def namespace(self, prefix):
 
     def namespaces(self):
         """ """
-        if False:
-            yield None
 
     # Optional Transactional methods
 

requirements.dev.txt

@@ -7,3 +7,6 @@ flake8
 doctest-ignore-unicode==0.1.2
 berkeleydb
 black==21.6b0
+flake8-black
+mypy
+types-setuptools

setup.cfg

@@ -33,3 +33,4 @@ python_version = 3.6
 warn_unused_configs = True
 ignore_missing_imports = True
 disallow_subclassing_any = False
+warn_unreachable = True

test/test_csv2rdf.py

@@ -1,5 +1,6 @@
 import subprocess
 import unittest
+import sys
 from os import remove
 from tempfile import mkstemp
 from pathlib import Path
@@ -11,7 +12,12 @@ def setUp(self):
 
     def test_csv2rdf_cli(self):
         completed = subprocess.run(
-            ["csv2rdf", str(self.REALESTATE_FILE_PATH)],
+            [
+                sys.executable,
+                "-m",
+                "rdflib.tools.csv2rdf",
+                str(self.REALESTATE_FILE_PATH),
+            ],
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             universal_newlines=True,
@@ -25,7 +31,14 @@ def test_csv2rdf_cli(self):
     def test_csv2rdf_cli_fileout(self):
         _, fname = mkstemp()
         completed = subprocess.run(
-            ["csv2rdf", "-o", fname, str(self.REALESTATE_FILE_PATH)],
+            [
+                sys.executable,
+                "-m",
+                "rdflib.tools.csv2rdf",
+                "-o",
+                fname,
+                str(self.REALESTATE_FILE_PATH),
+            ],
         )
         self.assertEqual(completed.returncode, 0)
         with open(fname) as f:

tox.ini

@@ -24,3 +24,12 @@ commands =
 deps =
 	-rrequirements.txt
 	-rrequirements.dev.txt
+
+[testenv:mypy]
+basepython =
+    python3.7
+commands =
+    {envpython} -m mypy rdflib --show-error-context --show-error-codes
+deps =
+	-rrequirements.txt
+	-rrequirements.dev.txt