diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..c1024fc56 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.tox +.venv +.mypy_cache +.git diff --git a/.drone.yml b/.drone.yml index 55d1c81ee..4abdb59c0 100644 --- a/.drone.yml +++ b/.drone.yml @@ -26,8 +26,9 @@ steps: - pip install --default-timeout 60 -r requirements.dev.txt - pip install --default-timeout 60 coveralls && export HAS_COVERALLS=1 - python setup.py install - - black --config black.toml --check ./rdflib | true + - black --config black.toml --check ./rdflib || true - flake8 --exit-zero rdflib + - mypy --show-error-context --show-error-codes rdflib - PYTHONWARNINGS=default nosetests --with-timer --timer-top-n 42 --with-coverage --cover-tests --cover-package=rdflib - coverage report --skip-covered - coveralls diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..177f450c8 --- /dev/null +++ b/.flake8 @@ -0,0 +1,6 @@ +# https://flake8.pycqa.org/en/latest/user/configuration.html +[flake8] +extend-ignore = + # E501: line too long + # Disabled so that black can control line length. + E501, diff --git a/Makefile b/Makefile index 171f76cfd..3b9b73f5b 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ tests: docker-compose -f docker-compose.tests.yml up test-runner docker-compose -f docker-compose.tests.yml down +.PHONY: build build: docker-compose -f docker-compose.tests.yml build @@ -14,3 +15,6 @@ reformat: check-format: black --config ./black.toml --check . + +check-types: + docker-compose -f docker-compose.tests.yml up check-types diff --git a/docker-compose.tests.yml b/docker-compose.tests.yml index c71e3143f..00c61d284 100644 --- a/docker-compose.tests.yml +++ b/docker-compose.tests.yml @@ -13,4 +13,12 @@ services: dockerfile: test/Dockerfile volumes: - .:/rdflib - command: ["/rdflib/run_tests_with_coverage_report.sh"] \ No newline at end of file + command: ["/rdflib/run_tests_with_coverage_report.sh"] + + check-types: + build: + context: . + dockerfile: test/Dockerfile + volumes: + - .:/rdflib + command: ["python", "-m", "mypy", "--show-error-context", "--show-error-codes" ,"/rdflib/rdflib"] diff --git a/docs/developers.rst b/docs/developers.rst index 52d8ea896..fed805056 100644 --- a/docs/developers.rst +++ b/docs/developers.rst @@ -15,6 +15,9 @@ Code should be formatted using `black `_. While not yet mandatory, it will be required in the future (6.0.0+).1 Use Black v21.6b1, with the black.toml config file provided. +Code should also pass `flake8 `_ linting +and `mypy `_ type checking. + Any new functionality being added to RDFLib should have doc tests and unit tests. Tests should be added for any functionality being changed that currently does not have any doc tests or unit tests. And all the @@ -28,7 +31,7 @@ Running tests ------------- Run tests with `nose `_: -.. code-block: bash +.. code-block:: bash $ pip install nose $ python run_tests.py @@ -42,10 +45,31 @@ Specific tests can either be run by module name or file name. For example:: $ python run_tests.py --tests rdflib.graph $ python run_tests.py --tests test/test_graph.py +Running static checks +--------------------- + +Check formatting with `black `_: + +.. code-block:: bash + + python -m black --config black.toml --check ./rdflib + +Check style and conventions with `flake8 `_: + +.. code-block:: bash + + python -m flake8 rdflib + +Check types with `mypy `_: + +.. code-block:: bash + + python -m mypy --show-error-context --show-error-codes rdflib + Writing documentation --------------------- -We use sphinx for generating HTML docs, see :ref:`docs` +We use sphinx for generating HTML docs, see :ref:`docs`. Continuous Integration ---------------------- diff --git a/rdflib/compare.py b/rdflib/compare.py index d3620276d..1349db8e9 100644 --- a/rdflib/compare.py +++ b/rdflib/compare.py @@ -454,15 +454,15 @@ def _traces( experimental = self._experimental_path(coloring_copy) experimental_score = set([c.key() for c in experimental]) if last_coloring: - generator = self._create_generator( + generator = self._create_generator( # type: ignore[unreachable] [last_coloring, experimental], generator ) last_coloring = experimental - if best_score is None or best_score < color_score: + if best_score is None or best_score < color_score: # type: ignore[unreachable] best = [refined_coloring] best_score = color_score best_experimental_score = experimental_score - elif best_score > color_score: + elif best_score > color_score: # type: ignore[unreachable] # prune this branch. if stats is not None: stats["prunings"] += 1 @@ -480,7 +480,7 @@ def _traces( d = [depth[0]] new_color = self._traces(coloring, stats=stats, depth=d) color_score = tuple([c.key() for c in refined_coloring]) - if best_score is None or color_score > best_score: + if best_score is None or color_score > best_score: # type: ignore[unreachable] discrete = [new_color] best_score = color_score best_depth = d[0] diff --git a/rdflib/extras/infixowl.py b/rdflib/extras/infixowl.py index d1de5d801..fbb28314f 100644 --- a/rdflib/extras/infixowl.py +++ b/rdflib/extras/infixowl.py @@ -331,9 +331,6 @@ def castToQName(x): if isinstance(thing, BNode): return thing.n3() return "<" + thing + ">" - logger.debug(list(store.objects(subject=thing, predicate=RDF.type))) - raise - return "[]" # +thing._id.encode('utf-8')+'' label = first(Class(thing, graph=store).label) if label: return label diff --git a/rdflib/graph.py b/rdflib/graph.py index 94627c3bb..dac700e24 100644 --- a/rdflib/graph.py +++ b/rdflib/graph.py @@ -1057,34 +1057,34 @@ def serialize( @overload def serialize( self, - destination: Union[str, BufferedIOBase], + destination: Union[str, BufferedIOBase, pathlib.PurePath], format: str = ..., base: Optional[str] = ..., encoding: Optional[str] = ..., **args, - ) -> None: + ) -> "Graph": ... # fallback @overload def serialize( self, - destination: Union[str, BufferedIOBase, None] = None, + destination: Union[str, BufferedIOBase, pathlib.PurePath, None] = None, format: str = "turtle", base: Optional[str] = None, encoding: Optional[str] = None, **args, - ) -> Optional[Union[bytes, str]]: + ) -> Union[bytes, str, "Graph"]: ... def serialize( self, - destination: Union[str, BufferedIOBase, None] = None, + destination: Union[str, BufferedIOBase, pathlib.PurePath, None] = None, format: str = "turtle", base: Optional[str] = None, encoding: Optional[str] = None, **args, - ) -> Optional[Union[bytes, str]]: + ) -> Union[bytes, str, "Graph"]: """Serialize the Graph to destination If destination is None serialize method returns the serialization as @@ -1123,10 +1123,10 @@ def serialize( location = cast(str, destination) scheme, netloc, path, params, _query, fragment = urlparse(location) if netloc != "": - print( + logger.warning( "WARNING: not saving as location" + "is not a local file reference" ) - return None + return self fd, name = tempfile.mkstemp() stream = os.fdopen(fd, "wb") serializer.serialize(stream, base=base, encoding=encoding, **args) @@ -1967,7 +1967,6 @@ def __iter__(self) -> Generator[DatasetQuad, None, None]: return self.quads((None, None, None, None)) - class QuotedGraph(Graph): """ Quoted Graphs are intended to implement Notation 3 formulae. They are diff --git a/rdflib/plugins/parsers/jsonld.py b/rdflib/plugins/parsers/jsonld.py index dcb5eca3e..77fa8b4dc 100644 --- a/rdflib/plugins/parsers/jsonld.py +++ b/rdflib/plugins/parsers/jsonld.py @@ -35,7 +35,8 @@ import warnings from rdflib.graph import ConjunctiveGraph -from rdflib.parser import Parser, URLInputSource +from rdflib.parser import URLInputSource +import rdflib.parser from rdflib.namespace import RDF, XSD from rdflib.term import URIRef, BNode, Literal @@ -78,12 +79,12 @@ pass -TYPE_TERM = Term(str(RDF.type), TYPE, VOCAB) +TYPE_TERM = Term(str(RDF.type), TYPE, VOCAB) # type: ignore[call-arg] ALLOW_LISTS_OF_LISTS = True # NOTE: Not allowed in JSON-LD 1.0 -class JsonLDParser(Parser): +class JsonLDParser(rdflib.parser.Parser): def __init__(self): super(JsonLDParser, self).__init__() diff --git a/rdflib/plugins/parsers/notation3.py b/rdflib/plugins/parsers/notation3.py index dd64f5820..3f8fdbac4 100755 --- a/rdflib/plugins/parsers/notation3.py +++ b/rdflib/plugins/parsers/notation3.py @@ -1168,7 +1168,7 @@ def uri_ref2(self, argstr, i, res): pfx, ln = qn[0] if pfx is None: assert 0, "not used?" - ns = self._baseURI + ADDED_HASH + ns = self._baseURI + ADDED_HASH # type: ignore[unreachable] else: try: ns = self._bindings[pfx] diff --git a/rdflib/plugins/parsers/rdfxml.py b/rdflib/plugins/parsers/rdfxml.py index 2571f7719..35986889c 100644 --- a/rdflib/plugins/parsers/rdfxml.py +++ b/rdflib/plugins/parsers/rdfxml.py @@ -2,7 +2,7 @@ An RDF/XML parser for RDFLib """ -from xml.sax import make_parser, handler +from xml.sax import make_parser, handler, xmlreader from xml.sax.handler import ErrorHandler from xml.sax.saxutils import quoteattr, escape @@ -266,7 +266,7 @@ def convert(self, name, qname, attrs): pass elif att in UNQUALIFIED: # if not RDFNS[att] in atts: - atts[RDFNS[att]] = v + atts[RDFNS[att]] = v # type: ignore[misc] else: atts[URIRef(att)] = v return name, atts @@ -474,7 +474,7 @@ def property_element_start(self, name, qname, attrs): o = URIRef(atts[att]) else: if datatype is not None: - language = None + language = None # type: ignore[unreachable] o = Literal(atts[att], language, datatype) if object is None: @@ -575,12 +575,12 @@ def literal_element_end(self, name, qname): self.parent.object += self.current.object + end -def create_parser(target, store): +def create_parser(target, store) -> xmlreader.XMLReader: parser = make_parser() try: # Workaround for bug in expatreader.py. Needed when # expatreader is trying to guess a prefix. - parser.start_namespace_decl("xml", "http://www.w3.org/XML/1998/namespace") + parser.start_namespace_decl("xml", "http://www.w3.org/XML/1998/namespace") # type: ignore[attr-defined] except AttributeError: pass # Not present in Jython (at least) parser.setFeature(handler.feature_namespaces, 1) diff --git a/rdflib/plugins/shared/jsonld/context.py b/rdflib/plugins/shared/jsonld/context.py index bfd41e23e..30bb02eab 100644 --- a/rdflib/plugins/shared/jsonld/context.py +++ b/rdflib/plugins/shared/jsonld/context.py @@ -544,4 +544,4 @@ def _get_source_id(self, source, key): "Term", "id, name, type, container, index, language, reverse, context," "prefix, protected", ) -Term.__new__.__defaults__ = (UNDEF, UNDEF, UNDEF, UNDEF, False, UNDEF, False, False) +Term.__new__.__defaults__ = (UNDEF, UNDEF, UNDEF, UNDEF, False, UNDEF, False, False) # type: ignore[attr-defined] diff --git a/rdflib/plugins/shared/jsonld/util.py b/rdflib/plugins/shared/jsonld/util.py index 50b34b54b..49fbdab43 100644 --- a/rdflib/plugins/shared/jsonld/util.py +++ b/rdflib/plugins/shared/jsonld/util.py @@ -1,12 +1,16 @@ # -*- coding: utf-8 -*- # https://github.com/RDFLib/rdflib-jsonld/blob/feature/json-ld-1.1/rdflib_jsonld/util.py +import typing as t -try: +if t.TYPE_CHECKING: import json +else: + try: + import json - assert json # workaround for pyflakes issue #13 -except ImportError: - import simplejson as json + assert json # workaround for pyflakes issue #13 + except ImportError: + import simplejson as json from os import sep from os.path import normpath diff --git a/rdflib/plugins/sparql/algebra.py b/rdflib/plugins/sparql/algebra.py index dd5216bbc..3177d47db 100644 --- a/rdflib/plugins/sparql/algebra.py +++ b/rdflib/plugins/sparql/algebra.py @@ -801,7 +801,7 @@ class ExpressionNotCoveredException(Exception): pass -def translateAlgebra(query_algebra: Query = None): +def translateAlgebra(query_algebra: Query): """ :param query_algebra: An algebra returned by the function call algebra.translateQuery(parse_tree). diff --git a/rdflib/plugins/sparql/sparql.py b/rdflib/plugins/sparql/sparql.py index 70f803d07..eedc9e746 100644 --- a/rdflib/plugins/sparql/sparql.py +++ b/rdflib/plugins/sparql/sparql.py @@ -78,7 +78,7 @@ def __len__(self) -> int: while d is not None: i += len(d._d) d = d.outer - return i + return i # type: ignore[unreachable] def __iter__(self): d = self diff --git a/rdflib/store.py b/rdflib/store.py index 9461eecff..915ca57ad 100644 --- a/rdflib/store.py +++ b/rdflib/store.py @@ -364,8 +364,6 @@ def namespace(self, prefix): def namespaces(self): """ """ - if False: - yield None # Optional Transactional methods diff --git a/requirements.dev.txt b/requirements.dev.txt index e0b9cf7af..fda1b49ec 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -7,3 +7,6 @@ flake8 doctest-ignore-unicode==0.1.2 berkeleydb black==21.6b0 +flake8-black +mypy +types-setuptools diff --git a/setup.cfg b/setup.cfg index a334b0516..bcb0bfde8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,3 +33,4 @@ python_version = 3.6 warn_unused_configs = True ignore_missing_imports = True disallow_subclassing_any = False +warn_unreachable = True diff --git a/test/test_csv2rdf.py b/test/test_csv2rdf.py index 16cf678c0..c3e9b92f2 100644 --- a/test/test_csv2rdf.py +++ b/test/test_csv2rdf.py @@ -1,5 +1,6 @@ import subprocess import unittest +import sys from os import remove from tempfile import mkstemp from pathlib import Path @@ -11,7 +12,12 @@ def setUp(self): def test_csv2rdf_cli(self): completed = subprocess.run( - ["csv2rdf", str(self.REALESTATE_FILE_PATH)], + [ + sys.executable, + "-m", + "rdflib.tools.csv2rdf", + str(self.REALESTATE_FILE_PATH), + ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, @@ -25,7 +31,14 @@ def test_csv2rdf_cli(self): def test_csv2rdf_cli_fileout(self): _, fname = mkstemp() completed = subprocess.run( - ["csv2rdf", "-o", fname, str(self.REALESTATE_FILE_PATH)], + [ + sys.executable, + "-m", + "rdflib.tools.csv2rdf", + "-o", + fname, + str(self.REALESTATE_FILE_PATH), + ], ) self.assertEqual(completed.returncode, 0) with open(fname) as f: diff --git a/tox.ini b/tox.ini index 2fd977166..0615a0b36 100644 --- a/tox.ini +++ b/tox.ini @@ -24,3 +24,12 @@ commands = deps = -rrequirements.txt -rrequirements.dev.txt + +[testenv:mypy] +basepython = + python3.7 +commands = + {envpython} -m mypy rdflib --show-error-context --show-error-codes +deps = + -rrequirements.txt + -rrequirements.dev.txt