Merge pull request #687 from prospector-dev/mypy

Add the types, test it using Mypy

.prospector.yml

@@ -14,6 +14,15 @@ mypy:
   options:
     ignore-missing-imports: true
     follow-imports: skip
+    check-untyped-defs: true
+    disallow-any-generics: true
+    disallow-untyped-defs: true
+    disallow-incomplete-defs: true
+    disallow-untyped-decorators: true
+    warn-unused-configs: true
+    warn-unused-ignores: true
+    warn-redundant-casts: true
+    extra-checks: true
 
 pylint:
   options:

prospector/autodetect.py

@@ -2,6 +2,7 @@
 import re
 import warnings
 from pathlib import Path
+from typing import Union
 
 from requirements_detector import find_requirements
 from requirements_detector.detect import RequirementsNotFound
@@ -19,7 +20,7 @@
 _IMPORT_MULTIPLE_REGEX = re.compile(r"^\s*import ([\._a-zA-Z0-9]+(, ){1})+")
 
 
-def find_from_imports(file_contents):
+def find_from_imports(file_contents: str) -> set[str]:
     names = set()
     for line in file_contents.split("\n"):
         match = _IMPORT_MULTIPLE_REGEX.match(line)
@@ -42,7 +43,7 @@ def find_from_imports(file_contents):
     return names
 
 
-def find_from_path(path: Path):
+def find_from_path(path: Path) -> set[str]:
     names = set()
 
     try:
@@ -68,29 +69,29 @@ def find_from_path(path: Path):
     return names
 
 
-def find_from_requirements(path):
+def find_from_requirements(path: Union[str, Path]) -> set[str]:
     reqs = find_requirements(path)
-    names = []
+    names: set[str] = set()
     for requirement in reqs:
         if requirement.name is not None and requirement.name.lower() in POSSIBLE_LIBRARIES:
-            names.append(requirement.name.lower())
+            names.add(requirement.name.lower())
     return names
 
 
-def autodetect_libraries(path):
+def autodetect_libraries(path: Union[str, Path]) -> set[str]:
     if os.path.isfile(path):
         path = os.path.dirname(path)
         if path == "":
             path = "."
 
-    libraries = []
+    libraries: set[str] = set()
 
     try:
         libraries = find_from_requirements(path)
     except RequirementsNotFound:
         pass
 
     if len(libraries) < len(POSSIBLE_LIBRARIES):
-        libraries = find_from_path(path)
+        libraries = find_from_path(Path(path))
 
     return libraries

prospector/blender.py

@@ -6,25 +6,29 @@
 # remove duplicates.
 import pkgutil
 from collections import defaultdict
+from pathlib import Path
+from typing import Optional
 
 import yaml
 
+from prospector.message import Message
+
 __all__ = (
     "blend",
     "BLEND_COMBOS",
 )
 
 
-def blend_line(messages, blend_combos=None):
+def blend_line(messages: list[Message], blend_combos: Optional[list[list[tuple[str, str]]]] = None) -> list[Message]:
     """
     Given a list of messages on the same line, blend them together so that we
     end up with one message per actual problem. Note that we can still return
     more than one message here if there are two or more different errors for
     the line.
     """
     blend_combos = blend_combos or BLEND_COMBOS
-    blend_lists = [[] for _ in range(len(blend_combos))]
-    blended = []
+    blend_lists: list[list[Message]] = [[] for _ in range(len(blend_combos))]
+    blended: list[Message] = []
 
     # first we split messages into each of the possible blendable categories
     # so that we have a list of lists of messages which can be blended together
@@ -72,18 +76,19 @@ def blend_line(messages, blend_combos=None):
         # it will appear in two blend_lists. Therefore we mark anything not taken from the blend list
         # as "consumed" and then filter later, to avoid such cases.
         for now_used in blend_list[1:]:
-            now_used.used = True
+            now_used.used = True  # type: ignore[attr-defined]
 
     return [m for m in blended if not getattr(m, "used", False)]
 
 
-def blend(messages, blend_combos=None):
+def blend(messages: list[Message], blend_combos: Optional[list[list[tuple[str, str]]]] = None) -> list[Message]:
     blend_combos = blend_combos or BLEND_COMBOS
 
     # group messages by file and then line number
-    msgs_grouped = defaultdict(lambda: defaultdict(list))
+    msgs_grouped: dict[Path, dict[int, list[Message]]] = defaultdict(lambda: defaultdict(list))
 
     for message in messages:
+        assert message.location.line is not None
         msgs_grouped[message.location.path][message.location.line].append(
             message,
         )
@@ -97,18 +102,20 @@ def blend(messages, blend_combos=None):
     return out
 
 
-def get_default_blend_combinations():
-    combos = yaml.safe_load(pkgutil.get_data(__name__, "blender_combinations.yaml"))
+def get_default_blend_combinations() -> list[list[tuple[str, str]]]:
+    blender_combinations = pkgutil.get_data(__name__, "blender_combinations.yaml")
+    assert blender_combinations is not None
+    combos = yaml.safe_load(blender_combinations)
     combos = combos.get("combinations", [])
 
-    defaults = []
+    defaults: list[list[tuple[str, str]]] = []
     for combo in combos:
         toblend = []
         for msg in combo:
             toblend += msg.items()
-        defaults.append(tuple(toblend))
+        defaults.append(toblend)
 
-    return tuple(defaults)
+    return defaults
 
 
 BLEND_COMBOS = get_default_blend_combinations()

prospector/config/__init__.py

@@ -2,7 +2,11 @@
 import re
 import sys
 from pathlib import Path
-from typing import Optional, Union
+from typing import Any, Callable, Optional, Union
+
+import setoptconf.config
+
+from prospector.finder import FileFinder
 
 try:  # Python >= 3.11
     import re._constants as sre_constants
@@ -36,17 +40,17 @@ def __init__(self, workdir: Optional[Path] = None):
         self.libraries = self._find_used_libraries(self.config, self.profile)
         self.tools_to_run = self._determine_tool_runners(self.config, self.profile)
         self.ignores = self._determine_ignores(self.config, self.profile, self.libraries)
-        self.configured_by: dict[str, str] = {}
+        self.configured_by: dict[str, Optional[Union[str, Path]]] = {}
         self.messages: list[Message] = []
 
-    def make_exclusion_filter(self):
+    def make_exclusion_filter(self) -> Callable[[Path], bool]:
         # Only close over the attributes required by the filter, rather
         # than the entire self, because ProspectorConfig can't be pickled
         # because of the config attribute, which would break parallel
         # pylint.
         ignores, workdir = self.ignores, self.workdir
 
-        def _filter(path: Path):
+        def _filter(path: Path) -> bool:
             for ignore in ignores:
                 # first figure out where the path is, relative to the workdir
                 # ignore-paths/patterns will usually be relative to a repository
@@ -60,19 +64,18 @@ def _filter(path: Path):
 
         return _filter
 
-    def get_tools(self, found_files):
+    def get_tools(self, found_files: FileFinder) -> list[tools.ToolBase]:
         self.configured_by = {}
         runners = []
         for tool_name in self.tools_to_run:
             tool = tools.TOOLS[tool_name]()
             config_result = tool.configure(self, found_files)
-            if config_result is None:
-                configured_by = None
-                messages = []
-            else:
-                configured_by, messages = config_result
-                if messages is None:
-                    messages = []
+            messages: list[Message] = []
+            configured_by = None
+            if config_result is not None:
+                configured_by, config_messages = config_result
+                if config_messages is not None:
+                    messages = list(config_messages)
 
             self.configured_by[tool_name] = configured_by
             self.messages += messages
@@ -93,26 +96,27 @@ def replace_deprecated_tool_names(self) -> list[str]:
         self.tools_to_run = replaced
         return deprecated_found
 
-    def get_output_report(self):
+    def get_output_report(self) -> list[tuple[str, list[str]]]:
         # Get the output formatter
+        output_report: list[tuple[str, list[str]]]
         if self.config.output_format is not None:
             output_report = self.config.output_format
         else:
-            output_report = [(self.profile.output_format, self.profile.output_target)]
+            output_report = [(self.profile.output_format, self.profile.output_target)]  # type: ignore[list-item]
 
         for index, report in enumerate(output_report):
             if not all(report):
                 output_report[index] = (report[0] or "grouped", report[1] or [])
 
         return output_report
 
-    def _configure_prospector(self):
+    def _configure_prospector(self) -> tuple[setoptconf.config.Configuration, dict[str, str]]:
         # first we will configure prospector as a whole
         mgr = cfg.build_manager()
         config = mgr.retrieve(*cfg.build_default_sources())
         return config, mgr.arguments
 
-    def _get_work_path(self, config, arguments) -> list[Path]:
+    def _get_work_path(self, config: setoptconf.config.Configuration, arguments: dict[str, str]) -> list[Path]:
         # Figure out what paths we're prospecting
         if config["path"]:
             paths = [Path(self.config["path"])]
@@ -122,7 +126,9 @@ def _get_work_path(self, config, arguments) -> list[Path]:
             paths = [Path.cwd()]
         return [p.resolve() for p in paths]
 
-    def _get_profile(self, workdir: Path, config):
+    def _get_profile(
+        self, workdir: Path, config: setoptconf.config.Configuration
+    ) -> tuple[ProspectorProfile, Optional[str]]:
         # Use the specified profiles
         profile_provided = False
         if len(config.profiles) > 0:
@@ -196,18 +202,18 @@ def _get_profile(self, workdir: Path, config):
             sys.exit(1)
         except ProfileNotFound as nfe:
             search_path = ":".join(map(str, nfe.profile_path))
-            profile = nfe.name.split(":")[0]
+            module_name = nfe.name.split(":")[0]
             sys.stderr.write(
                 f"""Failed to run:
 Could not find profile {nfe.name}.
-Search path: {search_path}, or in module 'prospector_profile_{profile}'
+Search path: {search_path}, or in module 'prospector_profile_{module_name}'
 """
             )
             sys.exit(1)
         else:
             return profile, strictness
 
-    def _find_used_libraries(self, config, profile):
+    def _find_used_libraries(self, config: setoptconf.config.Configuration, profile: ProspectorProfile) -> list[str]:
         libraries = []
 
         # Bring in adaptors that we automatically detect are needed
@@ -222,7 +228,7 @@ def _find_used_libraries(self, config, profile):
 
         return libraries
 
-    def _determine_tool_runners(self, config, profile):
+    def _determine_tool_runners(self, config: setoptconf.config.Configuration, profile: ProspectorProfile) -> list[str]:
         if config.tools is None:
             # we had no command line settings for an explicit list of
             # tools, so we use the defaults
@@ -255,7 +261,9 @@ def _determine_tool_runners(self, config, profile):
 
         return sorted(list(to_run))
 
-    def _determine_ignores(self, config, profile, libraries):
+    def _determine_ignores(
+        self, config: setoptconf.config.Configuration, profile: ProspectorProfile, libraries: list[str]
+    ) -> list[re.Pattern[str]]:
         # Grab ignore patterns from the options
         ignores = []
         for pattern in config.ignore_patterns + profile.ignore_patterns:
@@ -284,72 +292,72 @@ def _determine_ignores(self, config, profile, libraries):
 
         return ignores
 
-    def get_summary_information(self):
+    def get_summary_information(self) -> dict[str, Any]:
         return {
             "libraries": self.libraries,
             "strictness": self.strictness,
             "profiles": ", ".join(self.profile.list_profiles()),
             "tools": self.tools_to_run,
         }
 
-    def exit_with_zero_on_success(self):
+    def exit_with_zero_on_success(self) -> bool:
         return self.config.zero_exit
 
-    def get_disabled_messages(self, tool_name):
+    def get_disabled_messages(self, tool_name: str) -> list[str]:
         return self.profile.get_disabled_messages(tool_name)
 
-    def use_external_config(self, _):
+    def use_external_config(self, _: Any) -> bool:
         # Currently there is only one single global setting for whether to use
         # global config, but this could be extended in the future
         return not self.config.no_external_config
 
-    def tool_options(self, tool_name):
+    def tool_options(self, tool_name: str) -> dict[str, str]:
         tool = getattr(self.profile, tool_name, None)
         if tool is None:
             return {}
         return tool.get("options", {})
 
-    def external_config_location(self, tool_name):
+    def external_config_location(self, tool_name: str) -> Optional[Path]:
         return getattr(self.config, "%s_config_file" % tool_name, None)
 
     @property
-    def die_on_tool_error(self):
+    def die_on_tool_error(self) -> bool:
         return self.config.die_on_tool_error
 
     @property
-    def summary_only(self):
+    def summary_only(self) -> bool:
         return self.config.summary_only
 
     @property
-    def messages_only(self):
+    def messages_only(self) -> bool:
         return self.config.messages_only
 
     @property
-    def quiet(self):
+    def quiet(self) -> bool:
         return self.config.quiet
 
     @property
-    def blending(self):
+    def blending(self) -> bool:
         return self.config.blending
 
     @property
-    def absolute_paths(self):
+    def absolute_paths(self) -> bool:
         return self.config.absolute_paths
 
     @property
-    def max_line_length(self):
+    def max_line_length(self) -> int:
         return self.config.max_line_length
 
     @property
-    def include_tool_stdout(self):
+    def include_tool_stdout(self) -> bool:
         return self.config.include_tool_stdout
 
     @property
-    def direct_tool_stdout(self):
+    def direct_tool_stdout(self) -> bool:
         return self.config.direct_tool_stdout
 
     @property
-    def show_profile(self):
+    def show_profile(self) -> bool:
         return self.config.show_profile
 
     @property