From 2a0b1adab46bbe69e1d101bde24e33e7cd7d1276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Mon, 21 Oct 2024 11:59:02 +0200 Subject: [PATCH 1/2] Add the types, test it using Mypy --- .prospector.yml | 2 + prospector/autodetect.py | 17 ++-- prospector/blender.py | 29 ++++--- prospector/config/__init__.py | 81 ++++++++++--------- prospector/config/configuration.py | 9 ++- prospector/config/datatype.py | 2 +- prospector/encoding.py | 4 +- prospector/formatters/__init__.py | 2 +- prospector/formatters/base.py | 16 +++- prospector/formatters/base_summary.py | 2 +- prospector/formatters/emacs.py | 3 +- prospector/formatters/grouped.py | 6 +- prospector/formatters/json.py | 5 +- prospector/formatters/pylint.py | 2 +- prospector/formatters/text.py | 9 ++- prospector/formatters/vscode.py | 2 +- prospector/formatters/xunit.py | 2 +- prospector/formatters/yaml.py | 6 +- prospector/message.py | 17 ++-- prospector/profiles/exceptions.py | 16 ++-- prospector/profiles/profile.py | 77 ++++++++++-------- prospector/run.py | 30 +++---- prospector/suppression.py | 20 ++--- prospector/tools/__init__.py | 31 ++++--- prospector/tools/bandit/__init__.py | 22 +++-- prospector/tools/base.py | 14 +++- prospector/tools/dodgy/__init__.py | 10 ++- prospector/tools/mccabe/__init__.py | 22 +++-- prospector/tools/mypy/__init__.py | 30 ++++--- .../tools/profile_validator/__init__.py | 35 +++++--- prospector/tools/pycodestyle/__init__.py | 35 ++++---- prospector/tools/pydocstyle/__init__.py | 18 +++-- prospector/tools/pyflakes/__init__.py | 55 ++++++++----- prospector/tools/pylint/__init__.py | 57 ++++++++----- prospector/tools/pylint/collector.py | 9 +-- prospector/tools/pylint/linter.py | 12 ++- prospector/tools/pyright/__init__.py | 21 +++-- prospector/tools/pyroma/__init__.py | 16 ++-- prospector/tools/utils.py | 36 +++++---- prospector/tools/vulture/__init__.py | 36 ++++++--- 40 files changed, 501 insertions(+), 317 deletions(-) diff --git a/.prospector.yml b/.prospector.yml index 8579242b..7bf4eeff 100644 --- a/.prospector.yml +++ b/.prospector.yml @@ -14,6 +14,8 @@ mypy: options: ignore-missing-imports: true follow-imports: skip + disallow-untyped-defs: true + #strict: true pylint: options: diff --git a/prospector/autodetect.py b/prospector/autodetect.py index 93113edd..29473e26 100644 --- a/prospector/autodetect.py +++ b/prospector/autodetect.py @@ -2,6 +2,7 @@ import re import warnings from pathlib import Path +from typing import Union from requirements_detector import find_requirements from requirements_detector.detect import RequirementsNotFound @@ -19,7 +20,7 @@ _IMPORT_MULTIPLE_REGEX = re.compile(r"^\s*import ([\._a-zA-Z0-9]+(, ){1})+") -def find_from_imports(file_contents): +def find_from_imports(file_contents: str) -> set[str]: names = set() for line in file_contents.split("\n"): match = _IMPORT_MULTIPLE_REGEX.match(line) @@ -42,7 +43,7 @@ def find_from_imports(file_contents): return names -def find_from_path(path: Path): +def find_from_path(path: Path) -> set[str]: names = set() try: @@ -68,22 +69,22 @@ def find_from_path(path: Path): return names -def find_from_requirements(path): +def find_from_requirements(path: Union[str, Path]) -> set[str]: reqs = find_requirements(path) - names = [] + names: set[str] = set() for requirement in reqs: if requirement.name is not None and requirement.name.lower() in POSSIBLE_LIBRARIES: - names.append(requirement.name.lower()) + names.add(requirement.name.lower()) return names -def autodetect_libraries(path): +def autodetect_libraries(path: Union[str, Path]) -> set[str]: if os.path.isfile(path): path = os.path.dirname(path) if path == "": path = "." - libraries = [] + libraries: set[str] = set() try: libraries = find_from_requirements(path) @@ -91,6 +92,6 @@ def autodetect_libraries(path): pass if len(libraries) < len(POSSIBLE_LIBRARIES): - libraries = find_from_path(path) + libraries = find_from_path(Path(path)) return libraries diff --git a/prospector/blender.py b/prospector/blender.py index 3062f926..17dc741f 100644 --- a/prospector/blender.py +++ b/prospector/blender.py @@ -6,16 +6,20 @@ # remove duplicates. import pkgutil from collections import defaultdict +from pathlib import Path +from typing import Optional import yaml +from prospector.message import Message + __all__ = ( "blend", "BLEND_COMBOS", ) -def blend_line(messages, blend_combos=None): +def blend_line(messages: list[Message], blend_combos: Optional[list[list[tuple[str, str]]]] = None) -> list[Message]: """ Given a list of messages on the same line, blend them together so that we end up with one message per actual problem. Note that we can still return @@ -23,8 +27,8 @@ def blend_line(messages, blend_combos=None): the line. """ blend_combos = blend_combos or BLEND_COMBOS - blend_lists = [[] for _ in range(len(blend_combos))] - blended = [] + blend_lists: list[list[Message]] = [[] for _ in range(len(blend_combos))] + blended: list[Message] = [] # first we split messages into each of the possible blendable categories # so that we have a list of lists of messages which can be blended together @@ -72,18 +76,19 @@ def blend_line(messages, blend_combos=None): # it will appear in two blend_lists. Therefore we mark anything not taken from the blend list # as "consumed" and then filter later, to avoid such cases. for now_used in blend_list[1:]: - now_used.used = True + now_used.used = True # type: ignore[attr-defined] return [m for m in blended if not getattr(m, "used", False)] -def blend(messages, blend_combos=None): +def blend(messages: list[Message], blend_combos: Optional[list[list[tuple[str, str]]]] = None) -> list[Message]: blend_combos = blend_combos or BLEND_COMBOS # group messages by file and then line number - msgs_grouped = defaultdict(lambda: defaultdict(list)) + msgs_grouped: dict[Path, dict[int, list[Message]]] = defaultdict(lambda: defaultdict(list)) for message in messages: + assert message.location.line is not None msgs_grouped[message.location.path][message.location.line].append( message, ) @@ -97,18 +102,20 @@ def blend(messages, blend_combos=None): return out -def get_default_blend_combinations(): - combos = yaml.safe_load(pkgutil.get_data(__name__, "blender_combinations.yaml")) +def get_default_blend_combinations() -> list[list[tuple[str, str]]]: + blender_combinations = pkgutil.get_data(__name__, "blender_combinations.yaml") + assert blender_combinations is not None + combos = yaml.safe_load(blender_combinations) combos = combos.get("combinations", []) - defaults = [] + defaults: list[list[tuple[str, str]]] = [] for combo in combos: toblend = [] for msg in combo: toblend += msg.items() - defaults.append(tuple(toblend)) + defaults.append(toblend) - return tuple(defaults) + return defaults BLEND_COMBOS = get_default_blend_combinations() diff --git a/prospector/config/__init__.py b/prospector/config/__init__.py index 61125925..df29016e 100644 --- a/prospector/config/__init__.py +++ b/prospector/config/__init__.py @@ -2,7 +2,11 @@ import re import sys from pathlib import Path -from typing import Optional, Union +from typing import Any, Callable, Optional, Union + +import setoptconf.config + +from prospector.finder import FileFinder try: # Python >= 3.11 import re._constants as sre_constants @@ -36,17 +40,17 @@ def __init__(self, workdir: Optional[Path] = None): self.libraries = self._find_used_libraries(self.config, self.profile) self.tools_to_run = self._determine_tool_runners(self.config, self.profile) self.ignores = self._determine_ignores(self.config, self.profile, self.libraries) - self.configured_by: dict[str, str] = {} + self.configured_by: dict[str, Optional[Union[str, Path]]] = {} self.messages: list[Message] = [] - def make_exclusion_filter(self): + def make_exclusion_filter(self) -> Callable[[Path], bool]: # Only close over the attributes required by the filter, rather # than the entire self, because ProspectorConfig can't be pickled # because of the config attribute, which would break parallel # pylint. ignores, workdir = self.ignores, self.workdir - def _filter(path: Path): + def _filter(path: Path) -> bool: for ignore in ignores: # first figure out where the path is, relative to the workdir # ignore-paths/patterns will usually be relative to a repository @@ -60,19 +64,18 @@ def _filter(path: Path): return _filter - def get_tools(self, found_files): + def get_tools(self, found_files: FileFinder) -> list[tools.ToolBase]: self.configured_by = {} runners = [] for tool_name in self.tools_to_run: tool = tools.TOOLS[tool_name]() config_result = tool.configure(self, found_files) - if config_result is None: - configured_by = None - messages = [] - else: - configured_by, messages = config_result - if messages is None: - messages = [] + messages: list[Message] = [] + configured_by = None + if config_result is not None: + configured_by, config_messages = config_result + if config_messages is not None: + messages = list(config_messages) self.configured_by[tool_name] = configured_by self.messages += messages @@ -93,7 +96,7 @@ def replace_deprecated_tool_names(self) -> list[str]: self.tools_to_run = replaced return deprecated_found - def get_output_report(self): + def get_output_report(self) -> list[tuple[str, list[str]]]: # Get the output formatter if self.config.output_format is not None: output_report = self.config.output_format @@ -106,13 +109,13 @@ def get_output_report(self): return output_report - def _configure_prospector(self): + def _configure_prospector(self) -> tuple[setoptconf.config.Configuration, dict[str, str]]: # first we will configure prospector as a whole mgr = cfg.build_manager() config = mgr.retrieve(*cfg.build_default_sources()) return config, mgr.arguments - def _get_work_path(self, config, arguments) -> list[Path]: + def _get_work_path(self, config: setoptconf.config.Configuration, arguments: dict[str, str]) -> list[Path]: # Figure out what paths we're prospecting if config["path"]: paths = [Path(self.config["path"])] @@ -122,7 +125,9 @@ def _get_work_path(self, config, arguments) -> list[Path]: paths = [Path.cwd()] return [p.resolve() for p in paths] - def _get_profile(self, workdir: Path, config): + def _get_profile( + self, workdir: Path, config: setoptconf.config.Configuration + ) -> tuple[ProspectorProfile, Optional[str]]: # Use the specified profiles profile_provided = False if len(config.profiles) > 0: @@ -196,18 +201,18 @@ def _get_profile(self, workdir: Path, config): sys.exit(1) except ProfileNotFound as nfe: search_path = ":".join(map(str, nfe.profile_path)) - profile = nfe.name.split(":")[0] + module_name = nfe.name.split(":")[0] sys.stderr.write( f"""Failed to run: Could not find profile {nfe.name}. -Search path: {search_path}, or in module 'prospector_profile_{profile}' +Search path: {search_path}, or in module 'prospector_profile_{module_name}' """ ) sys.exit(1) else: return profile, strictness - def _find_used_libraries(self, config, profile): + def _find_used_libraries(self, config: setoptconf.config.Configuration, profile: ProspectorProfile) -> list[str]: libraries = [] # Bring in adaptors that we automatically detect are needed @@ -222,7 +227,7 @@ def _find_used_libraries(self, config, profile): return libraries - def _determine_tool_runners(self, config, profile): + def _determine_tool_runners(self, config: setoptconf.config.Configuration, profile: ProspectorProfile) -> list[str]: if config.tools is None: # we had no command line settings for an explicit list of # tools, so we use the defaults @@ -255,7 +260,9 @@ def _determine_tool_runners(self, config, profile): return sorted(list(to_run)) - def _determine_ignores(self, config, profile, libraries): + def _determine_ignores( + self, config: setoptconf.config.Configuration, profile: ProspectorProfile, libraries: list[str] + ) -> list[re.Pattern]: # Grab ignore patterns from the options ignores = [] for pattern in config.ignore_patterns + profile.ignore_patterns: @@ -284,7 +291,7 @@ def _determine_ignores(self, config, profile, libraries): return ignores - def get_summary_information(self): + def get_summary_information(self) -> dict[str, Any]: return { "libraries": self.libraries, "strictness": self.strictness, @@ -292,64 +299,64 @@ def get_summary_information(self): "tools": self.tools_to_run, } - def exit_with_zero_on_success(self): + def exit_with_zero_on_success(self) -> bool: return self.config.zero_exit - def get_disabled_messages(self, tool_name): + def get_disabled_messages(self, tool_name: str) -> list[str]: return self.profile.get_disabled_messages(tool_name) - def use_external_config(self, _): + def use_external_config(self, _: Any) -> bool: # Currently there is only one single global setting for whether to use # global config, but this could be extended in the future return not self.config.no_external_config - def tool_options(self, tool_name): + def tool_options(self, tool_name: str) -> dict[str, str]: tool = getattr(self.profile, tool_name, None) if tool is None: return {} return tool.get("options", {}) - def external_config_location(self, tool_name): + def external_config_location(self, tool_name: str) -> Optional[Path]: return getattr(self.config, "%s_config_file" % tool_name, None) @property - def die_on_tool_error(self): + def die_on_tool_error(self) -> bool: return self.config.die_on_tool_error @property - def summary_only(self): + def summary_only(self) -> bool: return self.config.summary_only @property - def messages_only(self): + def messages_only(self) -> bool: return self.config.messages_only @property - def quiet(self): + def quiet(self) -> bool: return self.config.quiet @property - def blending(self): + def blending(self) -> bool: return self.config.blending @property - def absolute_paths(self): + def absolute_paths(self) -> bool: return self.config.absolute_paths @property - def max_line_length(self): + def max_line_length(self) -> int: return self.config.max_line_length @property - def include_tool_stdout(self): + def include_tool_stdout(self) -> bool: return self.config.include_tool_stdout @property - def direct_tool_stdout(self): + def direct_tool_stdout(self) -> bool: return self.config.direct_tool_stdout @property - def show_profile(self): + def show_profile(self) -> bool: return self.config.show_profile @property diff --git a/prospector/config/configuration.py b/prospector/config/configuration.py index db0df3e1..9b0de966 100644 --- a/prospector/config/configuration.py +++ b/prospector/config/configuration.py @@ -1,4 +1,5 @@ import importlib.metadata +from typing import Optional import setoptconf as soc @@ -11,7 +12,7 @@ _VERSION = importlib.metadata.version("prospector") -def build_manager(): +def build_manager() -> soc.ConfigurationManager: manager = soc.ConfigurationManager("prospector") manager.add(soc.BooleanSetting("zero_exit", default=False)) @@ -76,7 +77,7 @@ def build_manager(): return manager -def build_default_sources(): +def build_default_sources() -> list[soc.Source]: sources = [ build_command_line_source(), soc.EnvironmentVariableSource(), @@ -98,7 +99,9 @@ def build_default_sources(): return sources -def build_command_line_source(prog=None, description="Performs static analysis of Python code"): +def build_command_line_source( + prog: Optional[str] = None, description: Optional[str] = "Performs static analysis of Python code" +) -> soc.CommandLineSource: parser_options = {} if prog is not None: parser_options["prog"] = prog diff --git a/prospector/config/datatype.py b/prospector/config/datatype.py index d3482e04..6c8394a6 100644 --- a/prospector/config/datatype.py +++ b/prospector/config/datatype.py @@ -6,7 +6,7 @@ class OutputChoice(Choice): - def sanitize(self, value): + def sanitize(self, value: str) -> tuple[str, list[str]]: parsed = re.split(r"[;:]", value) output_format, output_targets = parsed[0], parsed[1:] checked_targets = [] diff --git a/prospector/encoding.py b/prospector/encoding.py index e6308eac..5a599e12 100644 --- a/prospector/encoding.py +++ b/prospector/encoding.py @@ -7,8 +7,8 @@ # mypy complains with 'Incompatible return value type (got "str", expected "bytes")' -def read_py_file(filepath: Path): - # see https://docs.python.org/3/library/tokenize.html#tokenize.detect_encoding +def read_py_file(filepath: Path) -> str: + # See https://docs.python.org/3/library/tokenize.html#tokenize.detect_encoding # first just see if the file is properly encoded try: with open(filepath, "rb") as bfile_: diff --git a/prospector/formatters/__init__.py b/prospector/formatters/__init__.py index 2328bbc2..0399df65 100644 --- a/prospector/formatters/__init__.py +++ b/prospector/formatters/__init__.py @@ -4,7 +4,7 @@ __all__ = ("FORMATTERS", "Formatter") -FORMATTERS = { +FORMATTERS: dict[str, type[Formatter]] = { "json": json.JsonFormatter, "text": text.TextFormatter, "grouped": grouped.GroupedFormatter, diff --git a/prospector/formatters/base.py b/prospector/formatters/base.py index 839b144f..2ad06dc7 100644 --- a/prospector/formatters/base.py +++ b/prospector/formatters/base.py @@ -1,22 +1,30 @@ from abc import ABC, abstractmethod +from prospector.profiles.profile import ProspectorProfile + __all__ = ("Formatter",) from pathlib import Path -from typing import Optional +from typing import Any, Optional from prospector.message import Message class Formatter(ABC): - def __init__(self, summary, messages, profile, paths_relative_to: Optional[Path] = None): + def __init__( + self, + summary: dict[str, Any], + messages: list[Message], + profile: ProspectorProfile, + paths_relative_to: Optional[Path] = None, + ) -> None: self.summary = summary self.messages = messages self.profile = profile self.paths_relative_to = paths_relative_to @abstractmethod - def render(self, summary=True, messages=True, profile=False): + def render(self, summary: bool = True, messages: bool = True, profile: bool = False) -> str: raise NotImplementedError def _make_path(self, path: Path) -> str: @@ -26,7 +34,7 @@ def _make_path(self, path: Path) -> str: path = path.relative_to(self.paths_relative_to) return str(path) - def _message_to_dict(self, message: Message) -> dict: + def _message_to_dict(self, message: Message) -> dict[str, Any]: loc = { "path": self._make_path(message.location.path), "module": message.location.module, diff --git a/prospector/formatters/base_summary.py b/prospector/formatters/base_summary.py index be0ed1cb..125d31ff 100644 --- a/prospector/formatters/base_summary.py +++ b/prospector/formatters/base_summary.py @@ -20,7 +20,7 @@ class SummaryFormatter(Formatter): ("external_config", "External Config", None), ) - def render_summary(self): + def render_summary(self) -> str: output = [ "Check Information", "=================", diff --git a/prospector/formatters/emacs.py b/prospector/formatters/emacs.py index 38a28daf..e2b88b89 100644 --- a/prospector/formatters/emacs.py +++ b/prospector/formatters/emacs.py @@ -1,10 +1,11 @@ from prospector.formatters.text import TextFormatter +from prospector.message import Message __all__ = ("EmacsFormatter",) class EmacsFormatter(TextFormatter): - def render_message(self, message): + def render_message(self, message: Message) -> str: output = [ "%s:%s:%d:" % ( diff --git a/prospector/formatters/grouped.py b/prospector/formatters/grouped.py index e7445a20..7d876826 100644 --- a/prospector/formatters/grouped.py +++ b/prospector/formatters/grouped.py @@ -1,21 +1,23 @@ from collections import defaultdict from prospector.formatters.text import TextFormatter +from prospector.message import Message __all__ = ("GroupedFormatter",) class GroupedFormatter(TextFormatter): - def render_messages(self): + def render_messages(self) -> str: output = [ "Messages", "========", "", ] - groups = defaultdict(lambda: defaultdict(list)) + groups: dict[str, dict[int, list[Message]]] = defaultdict(lambda: defaultdict(list)) for message in self.messages: + assert message.location.line is not None groups[self._make_path(message.location.path)][message.location.line].append(message) for filename in sorted(groups.keys()): diff --git a/prospector/formatters/json.py b/prospector/formatters/json.py index aae87b32..c9bc9a83 100644 --- a/prospector/formatters/json.py +++ b/prospector/formatters/json.py @@ -1,5 +1,6 @@ import json from datetime import datetime +from typing import Any from prospector.formatters.base import Formatter @@ -7,8 +8,8 @@ class JsonFormatter(Formatter): - def render(self, summary=True, messages=True, profile=False): - output = {} + def render(self, summary: bool = True, messages: bool = True, profile: bool = False) -> str: + output: dict[str, Any] = {} if summary: # we need to slightly change the types and format diff --git a/prospector/formatters/pylint.py b/prospector/formatters/pylint.py index a2491fec..2030ad76 100644 --- a/prospector/formatters/pylint.py +++ b/prospector/formatters/pylint.py @@ -11,7 +11,7 @@ class PylintFormatter(SummaryFormatter): on top of pylint and prospector itself. """ - def render(self, summary=True, messages=True, profile=False): + def render(self, summary: bool = True, messages: bool = True, profile: bool = False) -> str: # this formatter will always ignore the summary and profile cur_loc = None output = [] diff --git a/prospector/formatters/text.py b/prospector/formatters/text.py index 66198fc8..e84673ce 100644 --- a/prospector/formatters/text.py +++ b/prospector/formatters/text.py @@ -1,4 +1,5 @@ from prospector.formatters.base_summary import SummaryFormatter +from prospector.message import Message __all__ = ("TextFormatter",) @@ -8,7 +9,7 @@ class TextFormatter(SummaryFormatter): - def render_message(self, message): + def render_message(self, message: Message) -> str: output = [] if message.location.module: @@ -31,7 +32,7 @@ def render_message(self, message): return "\n".join(output) - def render_messages(self): + def render_messages(self) -> str: output = [ "Messages", "========", @@ -44,12 +45,12 @@ def render_messages(self): return "\n".join(output) - def render_profile(self): + def render_profile(self) -> str: output = ["Profile", "=======", "", self.profile.as_yaml().strip()] return "\n".join(output) - def render(self, summary=True, messages=True, profile=False): + def render(self, summary: bool = True, messages: bool = True, profile: bool = False) -> str: output = [] if messages and self.messages: # if there are no messages, don't render an empty header output.append(self.render_messages()) diff --git a/prospector/formatters/vscode.py b/prospector/formatters/vscode.py index cbecabed..b552ef25 100644 --- a/prospector/formatters/vscode.py +++ b/prospector/formatters/vscode.py @@ -9,7 +9,7 @@ class VSCodeFormatter(SummaryFormatter): This formatter outputs messages in the same way as vscode prospector linter expects. """ - def render(self, summary=True, messages=True, profile=False): + def render(self, summary: bool = True, messages: bool = True, profile: bool = False) -> str: # this formatter will always ignore the summary and profile cur_loc = None output = [] diff --git a/prospector/formatters/xunit.py b/prospector/formatters/xunit.py index c815dee8..7671c156 100644 --- a/prospector/formatters/xunit.py +++ b/prospector/formatters/xunit.py @@ -10,7 +10,7 @@ class XunitFormatter(Formatter): to use Xunit and prospector itself. """ - def render(self, summary=True, messages=True, profile=False): + def render(self, summary: bool = True, messages: bool = True, profile: bool = False) -> str: xml_doc = Document() testsuite_el = xml_doc.createElement("testsuite") diff --git a/prospector/formatters/yaml.py b/prospector/formatters/yaml.py index 4b2e9f80..3d20c03f 100644 --- a/prospector/formatters/yaml.py +++ b/prospector/formatters/yaml.py @@ -1,3 +1,5 @@ +from typing import Any + import yaml from prospector.formatters.base import Formatter @@ -6,8 +8,8 @@ class YamlFormatter(Formatter): - def render(self, summary=True, messages=True, profile=False): - output = {} + def render(self, summary: bool = True, messages: bool = True, profile: bool = False) -> str: + output: dict[str, Any] = {} if summary: output["summary"] = self.summary diff --git a/prospector/message.py b/prospector/message.py index 658af52c..f0ac15d2 100644 --- a/prospector/message.py +++ b/prospector/message.py @@ -4,7 +4,12 @@ class Location: def __init__( - self, path: Union[Path, str], module: Optional[str], function: Optional[str], line: int, character: int + self, + path: Union[Path, str], + module: Optional[str], + function: Optional[str], + line: Optional[int], + character: Optional[int], ): if isinstance(path, Path): self._path = path @@ -18,7 +23,7 @@ def __init__( self.character = None if character == -1 else character @property - def path(self): + def path(self) -> Path: return self._path def absolute_path(self) -> Path: @@ -38,7 +43,7 @@ def __eq__(self, other: object) -> bool: return False return self._path == other._path and self.line == other.line and self.character == other.character - def __lt__(self, other: object) -> bool: + def __lt__(self, other: "Location") -> bool: if not isinstance(other, Location): raise ValueError if self._path == other._path: @@ -65,7 +70,7 @@ def __eq__(self, other: object) -> bool: return self.code == other.code return False - def __lt__(self, other) -> bool: + def __lt__(self, other: "Message") -> bool: if self.location == other.location: return self.code < other.code return self.location < other.location @@ -76,8 +81,8 @@ def make_tool_error_message( source: str, code: str, message: str, - line: int = 0, - character: int = 0, + line: Optional[int] = None, + character: Optional[int] = None, module: Optional[str] = None, function: Optional[str] = None, ) -> Message: diff --git a/prospector/profiles/exceptions.py b/prospector/profiles/exceptions.py index f0f75e93..84564875 100644 --- a/prospector/profiles/exceptions.py +++ b/prospector/profiles/exceptions.py @@ -1,10 +1,10 @@ class ProfileNotFound(Exception): - def __init__(self, name, profile_path): + def __init__(self, name: str, profile_path: str) -> None: super().__init__() self.name = name self.profile_path = profile_path - def __repr__(self): + def __repr__(self) -> str: return "Could not find profile {}; searched in {}".format( self.name, ":".join(self.profile_path), @@ -12,17 +12,17 @@ def __repr__(self): class CannotParseProfile(Exception): - def __init__(self, filepath, parse_error): + def __init__(self, filepath: str, parse_error: Exception) -> None: super().__init__() self.filepath = filepath self.parse_error = parse_error - def get_parse_message(self): + def get_parse_message(self) -> str: return "{}\n on line {} : char {}".format( - self.parse_error.problem, - self.parse_error.problem_mark.line, - self.parse_error.problem_mark.column, + self.parse_error.problem, # type: ignore[attr-defined] + self.parse_error.problem_mark.line, # type: ignore[attr-defined] + self.parse_error.problem_mark.column, # type: ignore[attr-defined] ) - def __repr__(self): + def __repr__(self) -> str: return "Could not parse profile found at %s - it is not valid YAML" % self.filepath diff --git a/prospector/profiles/profile.py b/prospector/profiles/profile.py index bbb7383a..6c848661 100644 --- a/prospector/profiles/profile.py +++ b/prospector/profiles/profile.py @@ -14,7 +14,7 @@ class ProspectorProfile: - def __init__(self, name: str, profile_dict: dict[str, Any], inherit_order: list[str]): + def __init__(self, name: str, profile_dict: dict[str, Any], inherit_order: list[str]) -> None: self.name = name self.inherit_order = inherit_order @@ -52,23 +52,23 @@ def __init__(self, name: str, profile_dict: dict[str, Any], inherit_order: list[ setattr(self, tool, conf) - def get_disabled_messages(self, tool_name): + def get_disabled_messages(self, tool_name: str) -> list[str]: disable = getattr(self, tool_name)["disable"] enable = getattr(self, tool_name)["enable"] return list(set(disable) - set(enable)) - def is_tool_enabled(self, name): + def is_tool_enabled(self, name: str) -> bool: enabled = getattr(self, name).get("run") if enabled is not None: return enabled # this is not explicitly enabled or disabled, so use the default return name in DEFAULT_TOOLS - def list_profiles(self): + def list_profiles(self) -> list[str]: # this profile is itself included return [str(profile) for profile in self.inherit_order] - def as_dict(self): + def as_dict(self) -> dict[str, Any]: out = { "ignore-paths": self.ignore_paths, "ignore-patterns": self.ignore_patterns, @@ -87,10 +87,10 @@ def as_dict(self): out[tool] = getattr(self, tool) return out - def as_json(self): + def as_json(self) -> str: return json.dumps(self.as_dict()) - def as_yaml(self): + def as_yaml(self) -> str: return yaml.safe_dump(self.as_dict()) @staticmethod @@ -99,7 +99,7 @@ def load( profile_path: list[Path], allow_shorthand: bool = True, forced_inherits: Optional[list[str]] = None, - ): + ) -> "ProspectorProfile": # First simply load all of the profiles and those that it explicitly inherits from data, inherits = _load_and_merge( name_or_path, @@ -110,12 +110,12 @@ def load( return ProspectorProfile(str(name_or_path), data, inherits) -def _is_valid_extension(filename): +def _is_valid_extension(filename: Union[str, Path]) -> bool: ext = os.path.splitext(filename)[1] return ext in (".yml", ".yaml") -def _load_content_package(name): +def _load_content_package(name: str) -> Optional[dict[str, Any]]: name_split = name.split(":", 1) module_name = f"prospector_profile_{name_split[0]}" file_names = ( @@ -138,10 +138,11 @@ def _load_content_package(name): try: return yaml.safe_load(data) or {} except yaml.parser.ParserError as parse_error: + assert used_name is not None raise CannotParseProfile(used_name, parse_error) from parse_error -def _load_content(name_or_path, profile_path): +def _load_content(name_or_path: Union[str, Path], profile_path: list[Path]) -> dict[str, Any]: filename = None optional = False @@ -165,14 +166,14 @@ def _load_content(name_or_path, profile_path): break if filename is None: - result = _load_content_package(name_or_path) + result = _load_content_package(str(name_or_path)) if result is not None: return result if optional: return {} - raise ProfileNotFound(name_or_path, profile_path) + raise ProfileNotFound(str(name_or_path), str(profile_path)) with codecs.open(filename) as fct: try: @@ -181,19 +182,19 @@ def _load_content(name_or_path, profile_path): raise CannotParseProfile(filename, parse_error) from parse_error -def _ensure_list(value): +def _ensure_list(value: Any) -> list[Any]: if isinstance(value, list): return value return [value] -def _simple_merge_dict(priority, base): +def _simple_merge_dict(priority: dict[str, Any], base: dict[str, Any]) -> dict[str, Any]: out = dict(base.items()) out.update(dict(priority.items())) return out -def _merge_tool_config(priority, base): +def _merge_tool_config(priority: dict[str, Any], base: dict[str, Any]) -> dict[str, Any]: out = dict(base.items()) # add options that are missing, but keep existing options from the priority dictionary @@ -208,10 +209,10 @@ def _merge_tool_config(priority, base): # anything enabled in the 'priority' dict is removed # from 'disabled' in the base dict and vice versa - base_disabled = base.get("disable") or [] - base_enabled = base.get("enable") or [] - pri_disabled = priority.get("disable") or [] - pri_enabled = priority.get("enable") or [] + base_disabled: list[Any] = base.get("disable") or [] + base_enabled: list[Any] = base.get("enable") or [] + pri_disabled: list[Any] = priority.get("disable") or [] + pri_enabled: list[Any] = priority.get("enable") or [] out["disable"] = list(set(pri_disabled) | (set(base_disabled) - set(pri_enabled))) out["enable"] = list(set(pri_enabled) | (set(base_enabled) - set(pri_disabled))) @@ -219,7 +220,7 @@ def _merge_tool_config(priority, base): return out -def _merge_profile_dict(priority: dict, base: dict) -> dict: +def _merge_profile_dict(priority: dict[str, Any], base: dict[str, Any]) -> dict[str, Any]: # copy the base dict into our output out = dict(base.items()) @@ -254,7 +255,7 @@ def _merge_profile_dict(priority: dict, base: dict) -> dict: return out -def _determine_strictness(profile_dict, inherits): +def _determine_strictness(profile_dict: dict[str, Any], inherits: list[str]) -> tuple[Optional[str], bool]: for profile in inherits: if profile.startswith("strictness_"): return None, False @@ -265,7 +266,7 @@ def _determine_strictness(profile_dict, inherits): return ("strictness_%s" % strictness), True -def _determine_pep8(profile_dict): +def _determine_pep8(profile_dict: dict[str, Any]) -> tuple[Optional[str], bool]: pep8 = profile_dict.get("pep8") if pep8 == "full": return "full_pep8", True @@ -276,28 +277,30 @@ def _determine_pep8(profile_dict): return None, False -def _determine_doc_warnings(profile_dict): +def _determine_doc_warnings(profile_dict: dict[str, Any]) -> tuple[Optional[str], bool]: doc_warnings = profile_dict.get("doc-warnings") if doc_warnings is None: return None, False return ("doc_warnings" if doc_warnings else "no_doc_warnings"), True -def _determine_test_warnings(profile_dict): +def _determine_test_warnings(profile_dict: dict[str, Any]) -> tuple[Optional[str], bool]: test_warnings = profile_dict.get("test-warnings") if test_warnings is None: return None, False return (None if test_warnings else "no_test_warnings"), True -def _determine_member_warnings(profile_dict): +def _determine_member_warnings(profile_dict: dict[str, Any]) -> tuple[Optional[str], bool]: member_warnings = profile_dict.get("member-warnings") if member_warnings is None: return None, False return ("member_warnings" if member_warnings else "no_member_warnings"), True -def _determine_implicit_inherits(profile_dict, already_inherits, shorthands_found): +def _determine_implicit_inherits( + profile_dict: dict[str, Any], already_inherits: list[str], shorthands_found: set[str] +) -> tuple[list[str], set[str]]: # Note: the ordering is very important here - the earlier items # in the list have precedence over the later items. The point of # the doc/test/pep8 profiles is usually to restore items which were @@ -324,7 +327,13 @@ def _determine_implicit_inherits(profile_dict, already_inherits, shorthands_foun return inherits, shorthands_found -def _append_profiles(name, profile_path, data, inherit_list, allow_shorthand=False): +def _append_profiles( + name: str, + profile_path: list[Path], + data: dict[Union[str, Path], Any], + inherit_list: list[str], + allow_shorthand: bool = False, +) -> tuple[dict[Union[str, Path], Any], list[str]]: new_data, new_il, _ = _load_profile(name, profile_path, allow_shorthand=allow_shorthand) data.update(new_data) inherit_list += new_il @@ -367,7 +376,7 @@ def _load_and_merge( # top of the inheritance tree to the bottom). This means that the lower down # values overwrite those from above, meaning that the initially provided profile # has precedence. - merged: dict = {} + merged: dict[str, Any] = {} for name in inherit_list[::-1]: priority = data[name] merged = _merge_profile_dict(priority, merged) @@ -375,7 +384,7 @@ def _load_and_merge( return merged, inherit_list -def _transform_legacy(profile_dict): +def _transform_legacy(profile_dict: dict[str, Any]) -> dict[str, Any]: """ After pep8 was renamed to pycodestyle, this pre-filter just moves profile config blocks using the old name to use the new name, merging if both are @@ -425,13 +434,13 @@ def _load_profile( already_loaded: Optional[list[Union[str, Path]]] = None, allow_shorthand: bool = True, forced_inherits: Optional[list[str]] = None, -): +) -> tuple[dict[Union[str, Path], Any], list[str], set[str]]: # recursively get the contents of the basic profile and those it inherits from base_contents = _load_content(name_or_path, profile_path) base_contents = _transform_legacy(base_contents) - inherit_order = [name_or_path] + inherit_order = [str(name_or_path)] shorthands_found = shorthands_found or set() already_loaded = already_loaded or [] @@ -448,7 +457,7 @@ def _load_profile( inherits += extra_inherits shorthands_found |= extra_shorthands - contents_dict = {name_or_path: base_contents} + contents_dict: dict[Union[str, Path], Any] = {name_or_path: base_contents} for inherit_profile in inherits: if inherit_profile in already_loaded: @@ -470,4 +479,4 @@ def _load_profile( # note: a new list is returned here rather than simply using inherit_order to give astroid a # clue about the type of the returned object, as otherwise it can recurse infinitely and crash, # this meaning that prospector does not run on prospector cleanly! - return contents_dict, list(inherit_order), shorthands_found + return contents_dict, inherit_order, shorthands_found diff --git a/prospector/run.py b/prospector/run.py index 796361b9..45f32140 100644 --- a/prospector/run.py +++ b/prospector/run.py @@ -1,10 +1,11 @@ +import argparse import codecs import os.path import sys import warnings from datetime import datetime from pathlib import Path -from typing import TextIO +from typing import Any, Optional, TextIO from prospector import blender, postfilter, tools from prospector.compat import is_relative_to @@ -19,12 +20,12 @@ class Prospector: - def __init__(self, config: ProspectorConfig): + def __init__(self, config: ProspectorConfig) -> None: self.config = config - self.summary = None + self.summary: Optional[dict[str, Any]] = None self.messages = config.messages - def process_messages(self, found_files, messages): + def process_messages(self, found_files: FileFinder, messages: list[Message]) -> list[Message]: if self.config.blending: messages = blender.blend(messages) @@ -38,10 +39,10 @@ def process_messages(self, found_files, messages): return postfilter.filter_messages(found_files.python_modules, messages) - def execute(self): + def execute(self) -> None: deprecated_names = self.config.replace_deprecated_tool_names() - summary = { + summary: dict[str, Any] = { "started": datetime.now(), } summary.update(self.config.get_summary_information()) @@ -124,25 +125,26 @@ def execute(self): summary["time_taken"] = "%0.2f" % delta.total_seconds() external_config = [] - for tool, configured_by in self.config.configured_by.items(): + for tool_name, configured_by in self.config.configured_by.items(): if configured_by is not None: - external_config.append((tool, configured_by)) + external_config.append((tool_name, configured_by)) if len(external_config) > 0: summary["external_config"] = ", ".join(["%s: %s" % info for info in external_config]) self.summary = summary self.messages = self.messages + messages - def get_summary(self): + def get_summary(self) -> Optional[dict[str, Any]]: return self.summary - def get_messages(self): + def get_messages(self) -> list[Message]: return self.messages - def print_messages(self): + def print_messages(self) -> None: output_reports = self.config.get_output_report() for report in output_reports: + assert self.summary is not None output_format, output_files = report self.summary["formatter"] = output_format @@ -161,7 +163,7 @@ def print_messages(self): with codecs.open(output_file, "w+") as target: self.write_to(formatter, target) - def write_to(self, formatter: Formatter, target: TextIO): + def write_to(self, formatter: Formatter, target: TextIO) -> None: # Produce the output target.write( formatter.render( @@ -173,7 +175,7 @@ def write_to(self, formatter: Formatter, target: TextIO): target.write("\n") -def get_parser(): +def get_parser() -> argparse.ArgumentParser: """ This is a helper method to return an argparse parser, to be used with the Sphinx argparse plugin for documentation. @@ -183,7 +185,7 @@ def get_parser(): return source.build_parser(manager.settings, None) -def main(): +def main() -> None: # Get our configuration config = ProspectorConfig() diff --git a/prospector/suppression.py b/prospector/suppression.py index c40d7021..018f5e82 100644 --- a/prospector/suppression.py +++ b/prospector/suppression.py @@ -24,7 +24,6 @@ import warnings from collections import defaultdict from pathlib import Path -from typing import List from prospector import encoding from prospector.exceptions import FatalProspectorException @@ -35,7 +34,7 @@ _PYLINT_SUPPRESSED_MESSAGE = re.compile(r"^Suppressed \'([a-z0-9-]+)\' \(from line \d+\)$") -def get_noqa_suppressions(file_contents): +def get_noqa_suppressions(file_contents: list[str]) -> tuple[bool, set[int]]: """ Finds all pep8/flake8 suppression messages @@ -64,9 +63,9 @@ def get_noqa_suppressions(file_contents): } -def _parse_pylint_informational(messages: List[Message]): - ignore_files = set() - ignore_messages: dict = defaultdict(lambda: defaultdict(list)) +def _parse_pylint_informational(messages: list[Message]) -> tuple[set[Path], dict[Path, dict[int, list[str]]]]: + ignore_files: set[Path] = set() + ignore_messages: dict[Path, dict[int, list[str]]] = defaultdict(lambda: defaultdict(list)) for message in messages: if message.source == "pylint": @@ -78,21 +77,24 @@ def _parse_pylint_informational(messages: List[Message]): raise FatalProspectorException(f"Could not parsed suppressed message from {message.message}") suppressed_code = match.group(1) line_dict = ignore_messages[message.location.path] + assert message.location.line is not None line_dict[message.location.line].append(suppressed_code) elif message.code == "file-ignored": ignore_files.add(message.location.path) return ignore_files, ignore_messages -def get_suppressions(filepaths: List[Path], messages): +def get_suppressions( + filepaths: list[Path], messages: list[Message] +) -> tuple[set[Path], dict[Path, set[int]], dict[Path, dict[int, set[tuple[str, str]]]]]: """ Given every message which was emitted by the tools, and the list of files to inspect, create a list of files to ignore, and a map of filepath -> line-number -> codes to ignore """ - paths_to_ignore = set() - lines_to_ignore: dict = defaultdict(set) - messages_to_ignore: dict = defaultdict(lambda: defaultdict(set)) + paths_to_ignore: set[Path] = set() + lines_to_ignore: dict[Path, set[int]] = defaultdict(set) + messages_to_ignore: dict[Path, dict[int, set[tuple[str, str]]]] = defaultdict(lambda: defaultdict(set)) # first deal with 'noqa' style messages for filepath in filepaths: diff --git a/prospector/tools/__init__.py b/prospector/tools/__init__.py index f04f7a44..7447ece6 100644 --- a/prospector/tools/__init__.py +++ b/prospector/tools/__init__.py @@ -1,16 +1,22 @@ -import importlib +from typing import TYPE_CHECKING, Any, Optional from prospector.exceptions import FatalProspectorException +from prospector.finder import FileFinder +from prospector.message import Message from prospector.tools.base import ToolBase from prospector.tools.dodgy import DodgyTool from prospector.tools.mccabe import McCabeTool +from prospector.tools.profile_validator import ProfileValidationTool # pylint: disable=cyclic-import from prospector.tools.pycodestyle import PycodestyleTool from prospector.tools.pydocstyle import PydocstyleTool from prospector.tools.pyflakes import PyFlakesTool from prospector.tools.pylint import PylintTool +if TYPE_CHECKING: + from prospector.config import ProspectorConfig -def _tool_not_available(name, install_option_name): + +def _tool_not_available(name: str, install_option_name: str) -> type[ToolBase]: class NotAvailableTool(ToolBase): """ Dummy tool class to return when a particular dependency is not found (such as mypy, or bandit) @@ -18,10 +24,10 @@ class NotAvailableTool(ToolBase): if the user tries to run prospector and specifies using the tool at which point an error is raised. """ - def configure(self, prospector_config, found_files): + def configure(self, prospector_config: "ProspectorConfig", found_files: FileFinder) -> None: pass - def run(self, _): + def run(self, _: Any) -> list[Message]: raise FatalProspectorException( f"\nCannot run tool {name} as support was not installed.\n" f"Please install by running 'pip install prospector[{install_option_name}]'\n\n" @@ -30,7 +36,12 @@ def run(self, _): return NotAvailableTool -def _optional_tool(name, package_name=None, tool_class_name=None, install_option_name=None): +def _optional_tool( + name: str, + package_name: Optional[str] = None, + tool_class_name: Optional[str] = None, + install_option_name: Optional[str] = None, +) -> type[ToolBase]: package_name = "prospector.tools.%s" % (package_name or name) tool_class_name = tool_class_name or f"{name.title()}Tool" install_option_name = install_option_name or f"with_{name}" @@ -45,20 +56,14 @@ def _optional_tool(name, package_name=None, tool_class_name=None, install_option return tool_class -def _profile_validator_tool(*args, **kwargs): - # bit of a hack to avoid a cyclic import... - mdl = importlib.import_module("prospector.tools.profile_validator") - return mdl.ProfileValidationTool(*args, **kwargs) - - -TOOLS = { +TOOLS: dict[str, type[ToolBase]] = { "dodgy": DodgyTool, "mccabe": McCabeTool, "pyflakes": PyFlakesTool, "pycodestyle": PycodestyleTool, "pylint": PylintTool, "pydocstyle": PydocstyleTool, - "profile-validator": _profile_validator_tool, + "profile-validator": ProfileValidationTool, "vulture": _optional_tool("vulture"), "pyroma": _optional_tool("pyroma"), "pyright": _optional_tool("pyright"), diff --git a/prospector/tools/bandit/__init__.py b/prospector/tools/bandit/__init__.py index 8aeeee77..930ee45a 100644 --- a/prospector/tools/bandit/__init__.py +++ b/prospector/tools/bandit/__init__.py @@ -1,14 +1,20 @@ +from typing import TYPE_CHECKING, Any + from bandit.cli.main import _get_profile, _init_extensions from bandit.core.config import BanditConfig from bandit.core.constants import RANKING from bandit.core.manager import BanditManager +from prospector.finder import FileFinder from prospector.message import Location, Message from prospector.tools.base import ToolBase +if TYPE_CHECKING: + from prospector.config import ProspectorConfig + class BanditTool(ToolBase): - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) self.manager = None self.profile = None @@ -17,22 +23,22 @@ def __init__(self, *args, **kwargs): self.severity = 0 self.confidence = 0 - def configure(self, prospector_config, _): + def configure(self, prospector_config: "ProspectorConfig", _: Any) -> None: options = prospector_config.tool_options("bandit") if "profile" in options: - self.profile = options["profile"] + self.profile = options["profile"] # type: ignore[assignment] if "config" in options: - self.config_file = options["config"] + self.config_file = options["config"] # type: ignore[assignment] if "severity" in options: - self.severity = options["severity"] + self.severity = options["severity"] # type: ignore[assignment] if not 0 <= self.severity <= 2: raise ValueError(f"severity {self.severity!r} must be between 0 and 2") if "confidence" in options: - self.confidence = options["confidence"] + self.confidence = options["confidence"] # type: ignore[assignment] if not 0 <= self.confidence <= 2: raise ValueError(f"confidence {self.confidence!r} must be between 0 and 2") @@ -43,7 +49,9 @@ def configure(self, prospector_config, _): self.manager = BanditManager(b_conf, None, profile=profile) - def run(self, found_files): + def run(self, found_files: FileFinder) -> list[Message]: + assert self.manager is not None + self.manager.files_list = sorted(found_files.files) self.manager.exclude_files = [] diff --git a/prospector/tools/base.py b/prospector/tools/base.py index f460d0a5..16b17f8a 100644 --- a/prospector/tools/base.py +++ b/prospector/tools/base.py @@ -1,12 +1,20 @@ from abc import ABC, abstractmethod -from typing import Iterable, List, Optional, Tuple +from collections.abc import Iterable +from pathlib import Path +from typing import TYPE_CHECKING, Optional, Union +from prospector.finder import FileFinder from prospector.message import Message +if TYPE_CHECKING: + from prospector.config import ProspectorConfig + class ToolBase(ABC): @abstractmethod - def configure(self, prospector_config, found_files) -> Tuple[str, Optional[Iterable[Message]]]: + def configure( + self, prospector_config: "ProspectorConfig", found_files: FileFinder + ) -> Optional[tuple[Optional[Union[str, Path]], Optional[Iterable[Message]]]]: """ Tools have their own way of being configured from configuration files on the current path - for example, a .pep8rc file. Prospector will use @@ -25,7 +33,7 @@ def configure(self, prospector_config, found_files) -> Tuple[str, Optional[Itera raise NotImplementedError @abstractmethod - def run(self, found_files) -> List[Message]: + def run(self, found_files: FileFinder) -> list[Message]: """ Actually run the tool and collect the various messages emitted by the tool. It is expected that this will convert whatever output of the tool into the diff --git a/prospector/tools/dodgy/__init__.py b/prospector/tools/dodgy/__init__.py index 32991af6..4a3d7161 100644 --- a/prospector/tools/dodgy/__init__.py +++ b/prospector/tools/dodgy/__init__.py @@ -1,5 +1,6 @@ import mimetypes from pathlib import Path +from typing import TYPE_CHECKING from dodgy.checks import check_file_contents @@ -8,18 +9,21 @@ from prospector.message import Location, Message from prospector.tools.base import ToolBase +if TYPE_CHECKING: + from prospector.config import ProspectorConfig -def module_from_path(path: Path): + +def module_from_path(path: Path) -> str: # TODO hacky... return ".".join(path.parts[1:-1] + (path.stem,)) class DodgyTool(ToolBase): - def configure(self, prospector_config, found_files): + def configure(self, prospector_config: "ProspectorConfig", found_files: FileFinder) -> None: # empty: just implementing to satisfy the ABC contract pass - def run(self, found_files: FileFinder): + def run(self, found_files: FileFinder) -> list[Message]: warnings = [] for filepath in found_files.files: mimetype = mimetypes.guess_type(str(filepath.absolute())) diff --git a/prospector/tools/mccabe/__init__.py b/prospector/tools/mccabe/__init__.py index 1bc8489e..7720cf48 100644 --- a/prospector/tools/mccabe/__init__.py +++ b/prospector/tools/mccabe/__init__.py @@ -1,28 +1,33 @@ import ast +from typing import TYPE_CHECKING, Any from mccabe import PathGraphingAstVisitor from prospector.encoding import CouldNotHandleEncoding, read_py_file +from prospector.finder import FileFinder from prospector.message import Location, Message, make_tool_error_message from prospector.tools.base import ToolBase +if TYPE_CHECKING: + from prospector.config import ProspectorConfig + __all__ = ("McCabeTool",) class McCabeTool(ToolBase): - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self.ignore_codes = () + self.ignore_codes: list[str] = [] self.max_complexity = 10 - def configure(self, prospector_config, _): + def configure(self, prospector_config: "ProspectorConfig", _: Any) -> None: self.ignore_codes = prospector_config.get_disabled_messages("mccabe") options = prospector_config.tool_options("mccabe") if "max-complexity" in options: - self.max_complexity = options["max-complexity"] + self.max_complexity = options["max-complexity"] # type: ignore[assignment] - def run(self, found_files): + def run(self, found_files: FileFinder) -> list[Message]: messages = [] for code_file in found_files.python_modules: @@ -38,7 +43,10 @@ def run(self, found_files): code_file, "mccabe", "MC0000", - message=f"Could not handle the encoding of this file: {err.encoding}", + message=( + "Could not handle the encoding of this file: " + f"{err.encoding}" # type: ignore[attr-defined] + ), ) ) continue @@ -77,5 +85,5 @@ def run(self, found_files): return self.filter_messages(messages) - def filter_messages(self, messages): + def filter_messages(self, messages: list[Message]) -> list[Message]: return [message for message in messages if message.code not in self.ignore_codes] diff --git a/prospector/tools/mypy/__init__.py b/prospector/tools/mypy/__init__.py index 56391251..f036799c 100644 --- a/prospector/tools/mypy/__init__.py +++ b/prospector/tools/mypy/__init__.py @@ -1,7 +1,9 @@ from multiprocessing import Process, Queue +from typing import TYPE_CHECKING, Any, Callable, Optional from mypy import api +from prospector.finder import FileFinder from prospector.message import Location, Message from prospector.tools import ToolBase @@ -9,16 +11,20 @@ from prospector.tools.exceptions import BadToolConfig +if TYPE_CHECKING: + from prospector.config import ProspectorConfig -def format_message(message): + +def format_message(message: str) -> Message: + character: Optional[int] try: - (path, line, char, err_type, err_msg) = message.split(":", 4) - line = int(line) - character = int(char) + (path, line_str, char_str, err_type, err_msg) = message.split(":", 4) + line = int(line_str) + character = int(char_str) except ValueError: try: - (path, line, err_type, err_msg) = message.split(":", 3) - line = int(line) + (path, line_str, err_type, err_msg) = message.split(":", 3) + line = int(line_str) character = None except ValueError: (path, err_type, err_msg) = message.split(":", 2) @@ -39,7 +45,7 @@ def format_message(message): ) -def _run_in_subprocess(q, cmd, paths): +def _run_in_subprocess(q: Queue, cmd: Callable[[list[str]], tuple[str, str]], paths: list[str]) -> None: """ This function exists only to be called by multiprocessing.Process as using lambda is forbidden @@ -48,16 +54,16 @@ def _run_in_subprocess(q, cmd, paths): class MypyTool(ToolBase): - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) self.checker = api self.options = ["--show-column-numbers", "--no-error-summary"] self.use_dmypy = False - def configure(self, prospector_config, _): + def configure(self, prospector_config: "ProspectorConfig", _: Any) -> None: options = prospector_config.tool_options("mypy") - self.use_dmypy = options.pop("use-dmypy", False) + self.use_dmypy = options.pop("use-dmypy", False) # type: ignore[assignment] # For backward compatibility if "follow-imports" not in options: @@ -81,13 +87,13 @@ def configure(self, prospector_config, _): raise BadToolConfig("mypy", f"The option {name} has an unsupported balue type: {type(value)}") - def run(self, found_files): + def run(self, found_files: FileFinder) -> list[Message]: paths = [str(path) for path in found_files.python_modules] paths.extend(self.options) if self.use_dmypy: # Due to dmypy messing with stdout/stderr we call it in a separate # process - q = Queue(1) + q: Queue[str] = Queue(1) p = Process(target=_run_in_subprocess, args=(q, self.checker.run_dmypy, ["run", "--"] + paths)) p.start() result = q.get() diff --git a/prospector/tools/profile_validator/__init__.py b/prospector/tools/profile_validator/__init__.py index cec0fe5e..0b24f8e3 100644 --- a/prospector/tools/profile_validator/__init__.py +++ b/prospector/tools/profile_validator/__init__.py @@ -1,5 +1,8 @@ import re from pathlib import Path +from typing import TYPE_CHECKING + +from prospector.tools.base import ToolBase try: # Python >= 3.11 import re._constants as sre_constants @@ -11,7 +14,10 @@ from prospector.finder import FileFinder from prospector.message import Location, Message from prospector.profiles import AUTO_LOADED_PROFILES -from prospector.tools import DEPRECATED_TOOL_NAMES, TOOLS, ToolBase, pyflakes + +if TYPE_CHECKING: + from prospector.config import ProspectorConfig + PROFILE_IS_EMPTY = "profile-is-empty" CONFIG_SETTING_SHOULD_BE_LIST = "should-be-list" @@ -26,7 +32,9 @@ __all__ = ("ProfileValidationTool",) -def _tool_names(with_deprecated: bool = True): +def _tool_names(with_deprecated: bool = True) -> list[str]: + from prospector.tools import DEPRECATED_TOOL_NAMES, TOOLS # pylint: disable=import-outside-toplevel + tools = list(TOOLS) if with_deprecated: tools += DEPRECATED_TOOL_NAMES.keys() @@ -49,27 +57,27 @@ class ProfileValidationTool(ToolBase): ) ALL_SETTINGS = LIST_SETTINGS + BOOL_SETTINGS + OTHER_SETTINGS - def __init__(self): + def __init__(self) -> None: self.to_check = set(AUTO_LOADED_PROFILES) - self.ignore_codes = () + self.ignore_codes: list[str] = [] - def configure(self, prospector_config, found_files): - for profile in prospector_config.config.profiles: + def configure(self, prospector_config: "ProspectorConfig", found_files: FileFinder) -> None: + for profile in prospector_config.config.profiles: # type: ignore[attr-defined] self.to_check.add(profile) self.ignore_codes = prospector_config.get_disabled_messages("profile-validator") - def validate(self, filepath: Path): # noqa + def validate(self, filepath: Path) -> list[Message]: # pylint: disable=too-many-locals # TODO: this should be broken down into smaller pieces - messages = [] + messages: list[Message] = [] with filepath.open() as profile_file: _file_contents = profile_file.read() parsed = yaml.safe_load(_file_contents) raw_contents = _file_contents.split("\n") - def add_message(code, message, setting): + def add_message(code: str, message: str, setting: str) -> None: if code in self.ignore_codes: return line = -1 @@ -77,9 +85,8 @@ def add_message(code, message, setting): if setting in fileline: line = number + 1 break - location = Location(filepath, None, None, line, 0, False) - message = Message("profile-validator", code, location, message) - messages.append(message) + location = Location(filepath, None, None, line, 0) + messages.append(Message("profile-validator", code, location, message)) if parsed is None: # this happens if a completely empty profile is found @@ -191,6 +198,8 @@ def add_message(code, message, setting): ) if "pyflakes" in parsed: + from prospector.tools import pyflakes # pylint: disable=import-outside-toplevel + for code in parsed["pyflakes"].get("enable", []) + parsed["pyflakes"].get("disable", []): if code in pyflakes.LEGACY_CODE_MAP: _legacy = pyflakes.LEGACY_CODE_MAP[code] @@ -202,7 +211,7 @@ def add_message(code, message, setting): return messages - def run(self, found_files: FileFinder): + def run(self, found_files: FileFinder) -> list[Message]: messages = [] for filepath in found_files.files: for possible in self.to_check: diff --git a/prospector/tools/pycodestyle/__init__.py b/prospector/tools/pycodestyle/__init__.py index ba3a227a..973bbf72 100644 --- a/prospector/tools/pycodestyle/__init__.py +++ b/prospector/tools/pycodestyle/__init__.py @@ -1,6 +1,9 @@ import codecs import os import re +from collections.abc import Iterable +from pathlib import Path +from typing import TYPE_CHECKING, Any, Optional, Union from pep8ext_naming import NamingChecker from pycodestyle import PROJECT_CONFIG, USER_CONFIG, BaseReport, StyleGuide, register_check @@ -9,15 +12,18 @@ from prospector.message import Location, Message from prospector.tools.base import ToolBase +if TYPE_CHECKING: + from prospector.config import ProspectorConfig + __all__ = ("PycodestyleTool",) class ProspectorReport(BaseReport): - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self._prospector_messages = [] + self._prospector_messages: list[Message] = [] - def error(self, line_number, offset, text, check): + def error(self, line_number: Optional[int], offset: int, text: str, check: str) -> None: code = super().error( line_number, offset, @@ -53,12 +59,12 @@ def error(self, line_number, offset, text, check): self._prospector_messages.append(message) - def get_messages(self): + def get_messages(self) -> list[Message]: return self._prospector_messages class ProspectorStyleGuide(StyleGuide): - def __init__(self, config, found_files, *args, **kwargs): + def __init__(self, config: "ProspectorConfig", found_files: FileFinder, *args: Any, **kwargs: Any) -> None: self._config = config self._files = found_files self._module_paths = found_files.python_modules @@ -68,7 +74,7 @@ def __init__(self, config, found_files, *args, **kwargs): super().__init__(*args, **kwargs) - def excluded(self, filename, parent=None): + def excluded(self, filename: str, parent: Optional[str] = None) -> bool: if super().excluded(filename, parent): return True @@ -82,11 +88,11 @@ def excluded(self, filename, parent=None): class PycodestyleTool(ToolBase): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.checker = None + checker: Optional[ProspectorStyleGuide] = None - def configure(self, prospector_config, found_files: FileFinder): + def configure( + self, prospector_config: "ProspectorConfig", found_files: FileFinder + ) -> Optional[tuple[Optional[str], Optional[Iterable[Message]]]]: # figure out if we should use a pre-existing config file # such as setup.cfg or tox.ini external_config = None @@ -97,18 +103,18 @@ def configure(self, prospector_config, found_files: FileFinder): if prospector_config.use_external_config("pycodestyle"): use_config = True - paths = [os.path.join(prospector_config.workdir, name) for name in PROJECT_CONFIG] + paths: list[Union[str, Path]] = [os.path.join(prospector_config.workdir, name) for name in PROJECT_CONFIG] paths.append(USER_CONFIG) ext_loc = prospector_config.external_config_location("pycodestyle") if ext_loc is not None: - paths = [ext_loc] + paths + paths = [ext_loc] + paths # type: ignore[assignment,operator] for conf_path in paths: if os.path.exists(conf_path) and os.path.isfile(conf_path): # this file exists - but does it have pep8 or pycodestyle config in it? # TODO: Remove this header = re.compile(r"\[(pep8|pycodestyle)\]") - with codecs.open(conf_path) as conf_file: + with codecs.open(str(conf_path)) as conf_file: if any(header.search(line) for line in conf_file.readlines()): external_config = conf_path break @@ -143,7 +149,8 @@ def configure(self, prospector_config, found_files: FileFinder): return configured_by, [] - def run(self, _): + def run(self, _: Any) -> list[Message]: + assert self.checker is not None report = self.checker.check_files() return report.get_messages() diff --git a/prospector/tools/pydocstyle/__init__.py b/prospector/tools/pydocstyle/__init__.py index b23d5a0d..53b3214e 100644 --- a/prospector/tools/pydocstyle/__init__.py +++ b/prospector/tools/pydocstyle/__init__.py @@ -1,3 +1,5 @@ +from typing import TYPE_CHECKING, Any + from pydocstyle.checker import AllError, ConventionChecker from prospector.encoding import CouldNotHandleEncoding, read_py_file @@ -5,19 +7,23 @@ from prospector.message import Location, Message, make_tool_error_message from prospector.tools.base import ToolBase +if TYPE_CHECKING: + from prospector.config import ProspectorConfig + + __all__ = ("PydocstyleTool",) class PydocstyleTool(ToolBase): - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self._code_files = [] - self.ignore_codes = () + self._code_files: list[str] = [] + self.ignore_codes: list[str] = [] - def configure(self, prospector_config, found_files): + def configure(self, prospector_config: "ProspectorConfig", found_files: FileFinder) -> None: self.ignore_codes = prospector_config.get_disabled_messages("pydocstyle") - def run(self, found_files: FileFinder): + def run(self, found_files: FileFinder) -> list[Message]: messages = [] checker = ConventionChecker() @@ -61,5 +67,5 @@ def run(self, found_files: FileFinder): return self.filter_messages(messages) - def filter_messages(self, messages): + def filter_messages(self, messages: list[Message]) -> list[Message]: return [message for message in messages if message.code not in self.ignore_codes] diff --git a/prospector/tools/pyflakes/__init__.py b/prospector/tools/pyflakes/__init__.py index 52b4fa84..ec83e143 100644 --- a/prospector/tools/pyflakes/__init__.py +++ b/prospector/tools/pyflakes/__init__.py @@ -1,15 +1,21 @@ +from typing import TYPE_CHECKING, Any, Optional + from pyflakes.api import checkPath +from pyflakes.messages import Message as FlakeMessage from pyflakes.reporter import Reporter +from prospector.finder import FileFinder from prospector.message import Location, Message from prospector.tools.base import ToolBase +if TYPE_CHECKING: + from prospector.config import ProspectorConfig __all__ = ("PyFlakesTool",) # Prospector uses the same pyflakes codes as flake8 defines, # see https://flake8.pycqa.org/en/latest/user/error-codes.html -# and https://gitlab.com/pycqa/flake8/-/blob/e817c63a/src/flake8/plugins/pyflakes.py +# and https://github.com/PyCQA/flake8/blob/e817c63a/src/flake8/plugins/pyflakes.py _MESSAGE_CODES = { "UnusedImport": "F401", "ImportShadowedByLoopVar": "F402", @@ -81,14 +87,22 @@ class ProspectorReporter(Reporter): - def __init__(self, ignore=None): + def __init__(self, ignore: Optional[list[str]] = None) -> None: super().__init__(None, None) - self._messages = [] + self._messages: list[Message] = [] self.ignore = ignore or () - def record_message( - self, filename=None, line=None, character=None, code=None, message=None - ): # pylint: disable=too-many-arguments + def record_message( # pylint: disable=too-many-arguments + self, + filename: str, + line: Optional[int] = None, + character: Optional[int] = None, + code: Optional[str] = None, + message: Optional[str] = None, + ) -> None: + assert message is not None + assert code is not None + code = code or "F999" if code in self.ignore: return @@ -100,15 +114,16 @@ def record_message( line=line, character=character, ) - message = Message( - source="pyflakes", - code=code, - location=location, - message=message, + self._messages.append( + Message( + source="pyflakes", + code=code, + location=location, + message=message, + ) ) - self._messages.append(message) - def unexpectedError(self, filename, msg): # noqa + def unexpectedError(self, filename: str, msg: str) -> None: self.record_message( filename=filename, code="F999", @@ -116,7 +131,7 @@ def unexpectedError(self, filename, msg): # noqa ) # pylint: disable=too-many-arguments - def syntaxError(self, filename, msg, lineno, offset, text): # noqa + def syntaxError(self, filename: str, msg: str, lineno: int, offset: int, text: str) -> None: self.record_message( filename=filename, line=lineno, @@ -125,7 +140,7 @@ def syntaxError(self, filename, msg, lineno, offset, text): # noqa message=msg, ) - def flake(self, message): + def flake(self, message: FlakeMessage) -> None: code = _MESSAGE_CODES.get(message.__class__.__name__, "F999") self.record_message( @@ -136,21 +151,21 @@ def flake(self, message): message=message.message % message.message_args, ) - def get_messages(self): + def get_messages(self) -> list[Message]: return self._messages class PyFlakesTool(ToolBase): - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self.ignore_codes = () + self.ignore_codes: list[str] = [] - def configure(self, prospector_config, _): + def configure(self, prospector_config: "ProspectorConfig", _: Any) -> None: ignores = prospector_config.get_disabled_messages("pyflakes") # convert old style to new self.ignore_codes = [LEGACY_CODE_MAP.get(code, code) for code in ignores] - def run(self, found_files): + def run(self, found_files: FileFinder) -> list[Message]: reporter = ProspectorReporter(ignore=self.ignore_codes) for filepath in found_files.python_modules: checkPath(str(filepath.absolute()), reporter) diff --git a/prospector/tools/pylint/__init__.py b/prospector/tools/pylint/__init__.py index 5ea6214a..21f02a5a 100644 --- a/prospector/tools/pylint/__init__.py +++ b/prospector/tools/pylint/__init__.py @@ -2,7 +2,9 @@ import re import sys from collections import defaultdict +from collections.abc import Iterable from pathlib import Path +from typing import TYPE_CHECKING, Any, Optional, Union from pylint.config import find_default_config_files from pylint.exceptions import UnknownMessageError @@ -14,6 +16,9 @@ from prospector.tools.pylint.collector import Collector from prospector.tools.pylint.linter import ProspectorLinter +if TYPE_CHECKING: + from prospector.config import ProspectorConfig + _UNUSED_WILDCARD_IMPORT_RE = re.compile(r"^Unused import(\(s\))? (.*) from wildcard import") @@ -26,12 +31,13 @@ class PylintTool(ToolBase): # be functions (they don't use the 'self' argument) but that would # make this module/class a bit ugly. - def __init__(self): - self._args = None - self._collector = self._linter = None - self._orig_sys_path = [] + def __init__(self) -> None: + self._args: Any = None + self._collector: Optional[Collector] = None + self._linter: Optional[ProspectorLinter] = None + self._orig_sys_path: list[str] = [] - def _prospector_configure(self, prospector_config, linter: ProspectorLinter): + def _prospector_configure(self, prospector_config: "ProspectorConfig", linter: ProspectorLinter) -> list[Message]: errors = [] if "django" in prospector_config.libraries: @@ -42,7 +48,7 @@ def _prospector_configure(self, prospector_config, linter: ProspectorLinter): linter.load_plugin_modules(["pylint_flask"]) profile_path = os.path.join(prospector_config.workdir, prospector_config.profile.name) - for plugin in prospector_config.profile.pylint.get("load-plugins", []): + for plugin in prospector_config.profile.pylint.get("load-plugins", []): # type: ignore[attr-defined] try: linter.load_plugin_modules([plugin]) except ImportError: @@ -88,11 +94,11 @@ def _prospector_configure(self, prospector_config, linter: ProspectorLinter): checker.set_option("max-line-length", max_line_length) return errors - def _error_message(self, filepath, message): + def _error_message(self, filepath: Union[str, Path], message: str) -> Message: location = Location(filepath, None, None, 0, 0) return Message("prospector", "config-problem", location, message) - def _pylintrc_configure(self, pylintrc, linter): + def _pylintrc_configure(self, pylintrc: Union[str, Path], linter: ProspectorLinter) -> list[Message]: errors = [] are_plugins_loaded = linter.config_from_file(pylintrc) if not are_plugins_loaded and hasattr(linter.config, "load_plugins"): @@ -103,7 +109,9 @@ def _pylintrc_configure(self, pylintrc, linter): errors.append(self._error_message(pylintrc, f"Could not load plugin {plugin}")) return errors - def configure(self, prospector_config, found_files: FileFinder): + def configure( + self, prospector_config: "ProspectorConfig", found_files: FileFinder + ) -> Optional[tuple[Optional[Union[str, Path]], Optional[Iterable[Message]]]]: extra_sys_path = found_files.make_syspath() check_paths = self._get_pylint_check_paths(found_files) @@ -128,7 +136,7 @@ def configure(self, prospector_config, found_files: FileFinder): self._linter = linter return configured_by, config_messages - def _set_path_finder(self, extra_sys_path: list[Path], pylint_options): + def _set_path_finder(self, extra_sys_path: list[Path], pylint_options: dict[str, Any]) -> None: # insert the target path into the system path to get correct behaviour self._orig_sys_path = sys.path if not pylint_options.get("use_pylint_default_path_finder"): @@ -166,17 +174,21 @@ def _get_pylint_check_paths(self, found_files: FileFinder) -> list[Path]: return sorted(check_paths) def _get_pylint_configuration( - self, check_paths: list[Path], linter: ProspectorLinter, prospector_config, pylint_options - ): + self, + check_paths: list[Path], + linter: ProspectorLinter, + prospector_config: "ProspectorConfig", + pylint_options: dict[str, Any], + ) -> tuple[list[Message], Optional[Union[Path, str]]]: self._args = check_paths linter.load_default_plugins() - config_messages = self._prospector_configure(prospector_config, linter) - configured_by = None + config_messages: list[Message] = self._prospector_configure(prospector_config, linter) + configured_by: Optional[Union[str, Path]] = None if prospector_config.use_external_config("pylint"): - # try to find a .pylintrc - pylintrc = pylint_options.get("config_file") + # Try to find a .pylintrc + pylintrc: Optional[Union[str, Path]] = pylint_options.get("config_file") external_config = prospector_config.external_config_location("pylint") pylintrc = pylintrc or external_config @@ -202,7 +214,7 @@ def _get_pylint_configuration( return config_messages, configured_by - def _combine_w0614(self, messages): + def _combine_w0614(self, messages: list[Message]) -> list[Message]: """ For the "unused import from wildcard import" messages, we want to combine all warnings about the same line into @@ -220,7 +232,9 @@ def _combine_w0614(self, messages): for location, message_list in by_loc.items(): names = [] for msg in message_list: - names.append(_UNUSED_WILDCARD_IMPORT_RE.match(msg.message).group(1)) + match_ = _UNUSED_WILDCARD_IMPORT_RE.match(msg.message) + assert match_ is not None + names.append(match_.group(1)) msgtxt = "Unused imports from wildcard import: %s" % ", ".join(names) combined_message = Message("pylint", "unused-wildcard-import", location, msgtxt) @@ -228,7 +242,7 @@ def _combine_w0614(self, messages): return out - def combine(self, messages): + def combine(self, messages: list[Message]) -> list[Message]: """ Combine repeated messages. @@ -242,7 +256,10 @@ def combine(self, messages): combined = self._combine_w0614(messages) return sorted(combined) - def run(self, found_files) -> list[Message]: + def run(self, found_files: FileFinder) -> list[Message]: + assert self._collector is not None + assert self._linter is not None + self._linter.check(self._args) sys.path = self._orig_sys_path diff --git a/prospector/tools/pylint/collector.py b/prospector/tools/pylint/collector.py index d5f65250..12a778f0 100644 --- a/prospector/tools/pylint/collector.py +++ b/prospector/tools/pylint/collector.py @@ -2,7 +2,7 @@ from typing import List from pylint.exceptions import UnknownMessageError -from pylint.message import Message as PylintMessage +from pylint.message import Message as PylintMessage, MessageDefinitionStore from pylint.reporters import BaseReporter from prospector.message import Location, Message @@ -11,10 +11,10 @@ class Collector(BaseReporter): name = "collector" - def __init__(self, message_store): + def __init__(self, message_store: MessageDefinitionStore) -> None: BaseReporter.__init__(self, output=StringIO()) self._message_store = message_store - self._messages = [] + self._messages: list[Message] = [] def handle_message(self, msg: PylintMessage) -> None: loc = Location(msg.abspath, msg.module, msg.obj, msg.line, msg.column) @@ -34,8 +34,5 @@ def handle_message(self, msg: PylintMessage) -> None: message = Message("pylint", msg_symbol, loc, msg.msg) self._messages.append(message) - def _display(self, layout) -> None: - pass - def get_messages(self) -> List[Message]: return self._messages diff --git a/prospector/tools/pylint/linter.py b/prospector/tools/pylint/linter.py index cb44bbbf..2405f006 100644 --- a/prospector/tools/pylint/linter.py +++ b/prospector/tools/pylint/linter.py @@ -1,30 +1,34 @@ +from collections.abc import Iterable from pathlib import Path +from typing import Any, Optional, Union from packaging import version as packaging_version from pylint import version as pylint_version from pylint.config.config_initialization import _config_initialization from pylint.lint import PyLinter +from prospector.finder import FileFinder + class UnrecognizedOptions(Exception): """Raised when an unrecognized option is found in the Pylint configuration.""" class ProspectorLinter(PyLinter): - def __init__(self, found_files, *args, **kwargs): + def __init__(self, found_files: FileFinder, *args: Any, **kwargs: Any) -> None: self._files = found_files # set up the standard PyLint linter PyLinter.__init__(self, *args, **kwargs) # Largely inspired by https://github.com/pylint-dev/pylint/blob/main/pylint/config/config_initialization.py#L26 - def config_from_file(self, config_file=None): + def config_from_file(self, config_file: Optional[Union[str, Path]] = None) -> bool: """Initialize the configuration from a file.""" _config_initialization(self, [], config_file=config_file) return True - def _expand_files(self, files_or_modules): + def _expand_files(self, files_or_modules: list[str]) -> Union[Iterable[Any], dict[str, Any]]: expanded = super()._expand_files(files_or_modules) - filtered = {} + filtered: dict[str, Any] = {} # PyLinter._expand_files returns dict since 2.15.7. if packaging_version.parse(pylint_version) > packaging_version.parse("2.15.6"): for module, expanded_module in expanded.items(): diff --git a/prospector/tools/pyright/__init__.py b/prospector/tools/pyright/__init__.py index 637b0c8f..5e930bf8 100644 --- a/prospector/tools/pyright/__init__.py +++ b/prospector/tools/pyright/__init__.py @@ -1,14 +1,21 @@ import json import subprocess +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any, Optional import pyright +from prospector.finder import FileFinder from prospector.message import Location, Message from prospector.tools import ToolBase +from prospector.tools.exceptions import BadToolConfig + +if TYPE_CHECKING: + from prospector.config import ProspectorConfig + __all__ = ("PyrightTool",) -from prospector.tools.exceptions import BadToolConfig VALID_OPTIONS = [ "level", @@ -21,7 +28,7 @@ ] -def format_messages(json_encoded): +def format_messages(json_encoded: str) -> list[Message]: json_decoded = json.loads(json_encoded) diagnostics = json_decoded.get("generalDiagnostics", []) messages = [] @@ -42,12 +49,14 @@ def format_messages(json_encoded): class PyrightTool(ToolBase): - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) self.checker = pyright self.options = ["--outputjson"] - def configure(self, prospector_config, _): + def configure( # pylint: disable=useless-return + self, prospector_config: "ProspectorConfig", _: Any + ) -> Optional[tuple[str, Optional[Iterable[Message]]]]: options = prospector_config.tool_options("pyright") for option_key in options.keys(): @@ -80,7 +89,9 @@ def configure(self, prospector_config, _): if venv_path: self.options.extend(["--venv-path", venv_path]) - def run(self, found_files): + return None + + def run(self, found_files: FileFinder) -> list[Message]: paths = [str(path) for path in found_files.python_modules] paths.extend(self.options) result = self.checker.run(*paths, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) diff --git a/prospector/tools/pyroma/__init__.py b/prospector/tools/pyroma/__init__.py index 5765f046..c4906275 100644 --- a/prospector/tools/pyroma/__init__.py +++ b/prospector/tools/pyroma/__init__.py @@ -1,5 +1,6 @@ import logging -from typing import TYPE_CHECKING, List +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any, Optional from prospector.finder import FileFinder from prospector.message import Location, Message @@ -48,7 +49,7 @@ PYROMA_CODES = {} -def _copy_codes(): +def _copy_codes() -> None: for name, code in PYROMA_ALL_CODES.items(): if hasattr(ratings, name): PYROMA_CODES[getattr(ratings, name)] = code @@ -60,14 +61,17 @@ def _copy_codes(): class PyromaTool(ToolBase): - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) - self.ignore_codes = () + self.ignore_codes: list[str] = [] - def configure(self, prospector_config: "ProspectorConfig", found_files: FileFinder): + def configure( # pylint: disable=useless-return + self, prospector_config: "ProspectorConfig", found_files: FileFinder + ) -> Optional[tuple[str, Optional[Iterable[Message]]]]: self.ignore_codes = prospector_config.get_disabled_messages("pyroma") + return None - def run(self, found_files: FileFinder) -> List[Message]: + def run(self, found_files: FileFinder) -> list[Message]: messages = [] for directory in found_files.directories: # just list directories which are not ignored, but find any `setup.py` ourselves diff --git a/prospector/tools/utils.py b/prospector/tools/utils.py index e973082e..0f446fa8 100644 --- a/prospector/tools/utils.py +++ b/prospector/tools/utils.py @@ -1,47 +1,55 @@ import sys +from io import TextIOWrapper +from typing import Optional -class CaptureStream: - def __init__(self): +class CaptureStream(TextIOWrapper): + def __init__(self) -> None: self.contents = "" - def write(self, text): + def write(self, text: str, /) -> int: self.contents += text + return len(text) - def close(self): + def close(self) -> None: pass - def flush(self): + def flush(self) -> None: pass class CaptureOutput: - def __init__(self, hide): + _prev_streams = None + stdout: Optional[TextIOWrapper] = None + stderr: Optional[TextIOWrapper] = None + + def __init__(self, hide: bool) -> None: self.hide = hide - self._prev_streams = None - self.stdout, self.stderr = None, None - def __enter__(self): + def __enter__(self) -> "CaptureOutput": if self.hide: - self._prev_streams = [ + self._prev_streams = ( sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__, - ] + ) self.stdout = CaptureStream() self.stderr = CaptureStream() sys.stdout, sys.__stdout__ = self.stdout, self.stdout # type: ignore[misc] sys.stderr, sys.__stderr__ = self.stderr, self.stderr # type: ignore[misc] return self - def get_hidden_stdout(self): + def get_hidden_stdout(self) -> str: + assert isinstance(self.stdout, CaptureStream) return self.stdout.contents - def get_hidden_stderr(self): + def get_hidden_stderr(self) -> str: + assert isinstance(self.stderr, CaptureStream) return self.stderr.contents - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__(self, exc_type: type, exc_val: Exception, exc_tb: type) -> None: if self.hide: + assert self._prev_streams is not None sys.stdout, sys.stderr, sys.__stdout__, sys.__stderr__ = self._prev_streams # type: ignore[misc] del self._prev_streams diff --git a/prospector/tools/vulture/__init__.py b/prospector/tools/vulture/__init__.py index b40787d8..f4de8c3d 100644 --- a/prospector/tools/vulture/__init__.py +++ b/prospector/tools/vulture/__init__.py @@ -1,19 +1,27 @@ +from collections.abc import Iterable +from pathlib import Path +from typing import TYPE_CHECKING, Any, Optional + from vulture import Vulture from prospector.encoding import CouldNotHandleEncoding, read_py_file +from prospector.finder import FileFinder from prospector.message import Location, Message, make_tool_error_message from prospector.tools.base import ToolBase +if TYPE_CHECKING: + from prospector.config import ProspectorConfig + class ProspectorVulture(Vulture): - def __init__(self, found_files): + def __init__(self, found_files: FileFinder) -> None: Vulture.__init__(self, verbose=False) self._files = found_files - self._internal_messages = [] - self.file = None - self.filename = None + self._internal_messages: list[Message] = [] + self.file: Optional[Path] = None + self.filename: Optional[Path] = None - def scavenge(self, _=None, __=None): + def scavenge(self, _: Any = None, __: Any = None) -> None: # The argument is a list of paths, but we don't care # about that as we use the found_files object. The # argument is here to explicitly acknowledge that we @@ -27,7 +35,10 @@ def scavenge(self, _=None, __=None): module, "vulture", "V000", - message=f"Could not handle the encoding of this file: {err.encoding}", + message=( + "Could not handle the encoding of this file: " + f"{err.encoding}" # type: ignore[attr-defined] + ), ) ) continue @@ -38,7 +49,7 @@ def scavenge(self, _=None, __=None): except TypeError: self.scan(module_string) - def get_messages(self): + def get_messages(self) -> list[Message]: all_items = ( ("unused-function", "Unused function %s", self.unused_funcs), ("unused-property", "Unused property %s", self.unused_props), @@ -66,15 +77,18 @@ def get_messages(self): class VultureTool(ToolBase): - def __init__(self): + def __init__(self) -> None: ToolBase.__init__(self) self._vulture = None - self.ignore_codes = () + self.ignore_codes: list[str] = [] - def configure(self, prospector_config, found_files): + def configure( # pylint: disable=useless-return + self, prospector_config: "ProspectorConfig", found_files: FileFinder + ) -> Optional[tuple[Optional[str], Optional[Iterable[Message]]]]: self.ignore_codes = prospector_config.get_disabled_messages("vulture") + return None - def run(self, found_files): + def run(self, found_files: FileFinder) -> list[Message]: vulture = ProspectorVulture(found_files) vulture.scavenge() return [message for message in vulture.get_messages() if message.code not in self.ignore_codes] From 6afb3fcd56310f4a47305051eb0648780d6ba59c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Brunner?= Date: Mon, 21 Oct 2024 20:37:58 +0200 Subject: [PATCH 2/2] Add some more mypy checks --- .prospector.yml | 9 ++++++++- prospector/config/__init__.py | 5 +++-- prospector/finder.py | 2 +- prospector/profiles/profile.py | 2 +- prospector/tools/mypy/__init__.py | 4 +++- prospector/tools/profile_validator/__init__.py | 2 +- 6 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.prospector.yml b/.prospector.yml index 7bf4eeff..8a768836 100644 --- a/.prospector.yml +++ b/.prospector.yml @@ -14,8 +14,15 @@ mypy: options: ignore-missing-imports: true follow-imports: skip + check-untyped-defs: true + disallow-any-generics: true disallow-untyped-defs: true - #strict: true + disallow-incomplete-defs: true + disallow-untyped-decorators: true + warn-unused-configs: true + warn-unused-ignores: true + warn-redundant-casts: true + extra-checks: true pylint: options: diff --git a/prospector/config/__init__.py b/prospector/config/__init__.py index df29016e..9d12905b 100644 --- a/prospector/config/__init__.py +++ b/prospector/config/__init__.py @@ -98,10 +98,11 @@ def replace_deprecated_tool_names(self) -> list[str]: def get_output_report(self) -> list[tuple[str, list[str]]]: # Get the output formatter + output_report: list[tuple[str, list[str]]] if self.config.output_format is not None: output_report = self.config.output_format else: - output_report = [(self.profile.output_format, self.profile.output_target)] + output_report = [(self.profile.output_format, self.profile.output_target)] # type: ignore[list-item] for index, report in enumerate(output_report): if not all(report): @@ -262,7 +263,7 @@ def _determine_tool_runners(self, config: setoptconf.config.Configuration, profi def _determine_ignores( self, config: setoptconf.config.Configuration, profile: ProspectorProfile, libraries: list[str] - ) -> list[re.Pattern]: + ) -> list[re.Pattern[str]]: # Grab ignore patterns from the options ignores = [] for pattern in config.ignore_patterns + profile.ignore_patterns: diff --git a/prospector/finder.py b/prospector/finder.py index 978fd769..19562308 100644 --- a/prospector/finder.py +++ b/prospector/finder.py @@ -18,7 +18,7 @@ class FileFinder: is basically to know which files to pass to which tools to be inspected. """ - def __init__(self, *provided_paths: Path, exclusion_filters: Optional[Iterable[Callable]] = None): + def __init__(self, *provided_paths: Path, exclusion_filters: Optional[Iterable[Callable[[Path], bool]]] = None): """ :param provided_paths: A list of Path objects to search for files and modules - can be either directories or files diff --git a/prospector/profiles/profile.py b/prospector/profiles/profile.py index 6c848661..7ffe3831 100644 --- a/prospector/profiles/profile.py +++ b/prospector/profiles/profile.py @@ -58,7 +58,7 @@ def get_disabled_messages(self, tool_name: str) -> list[str]: return list(set(disable) - set(enable)) def is_tool_enabled(self, name: str) -> bool: - enabled = getattr(self, name).get("run") + enabled: Optional[bool] = getattr(self, name).get("run") if enabled is not None: return enabled # this is not explicitly enabled or disabled, so use the default diff --git a/prospector/tools/mypy/__init__.py b/prospector/tools/mypy/__init__.py index f036799c..1832a849 100644 --- a/prospector/tools/mypy/__init__.py +++ b/prospector/tools/mypy/__init__.py @@ -45,7 +45,9 @@ def format_message(message: str) -> Message: ) -def _run_in_subprocess(q: Queue, cmd: Callable[[list[str]], tuple[str, str]], paths: list[str]) -> None: +def _run_in_subprocess( + q: "Queue[tuple[str, str]]", cmd: Callable[[list[str]], tuple[str, str]], paths: list[str] +) -> None: """ This function exists only to be called by multiprocessing.Process as using lambda is forbidden diff --git a/prospector/tools/profile_validator/__init__.py b/prospector/tools/profile_validator/__init__.py index 0b24f8e3..a92e225d 100644 --- a/prospector/tools/profile_validator/__init__.py +++ b/prospector/tools/profile_validator/__init__.py @@ -62,7 +62,7 @@ def __init__(self) -> None: self.ignore_codes: list[str] = [] def configure(self, prospector_config: "ProspectorConfig", found_files: FileFinder) -> None: - for profile in prospector_config.config.profiles: # type: ignore[attr-defined] + for profile in prospector_config.config.profiles: self.to_check.add(profile) self.ignore_codes = prospector_config.get_disabled_messages("profile-validator")