Exclude development tools from unused dependencies detection (#365)

* Add default ignore_unused and update pyproject.toml file

* Update unit test for cmd_lines

* Update tests for real projects

* Add tests for check unused action with default ignore option

* Apply review: change constant variable to UPPER_CASE

* Update unit test with other module while testing overriden cases

* Update DEFAULT_IGNORE_UNUSED and categorize it into different groups

* Bring in colorama as a transitive dependency, exclude it from unused

* Update poetry.lock

* Apply review: modify DEFAULT_IGNORE_UNUSED list

* Add ignore_unused to pyproject.toml to pass self_test

* Update README

* Remove default_ignore_unused_list.md and link to source code

* Exclude sphinx-* in DEFAULT_IGNORE_UNUSED list

* Update --help output for --ignore-unused option

README.md

@@ -307,10 +307,16 @@ declared or not, but you can ask for them to be ignored with the
 Conversely, there may be dependencies that you have declared without intending
 to `import` them. This is often the case for developer tools like Black or Mypy
 that are part of your project's development environment.
-FawltyDeps cannot automatically tell which of your declared dependencies are
-meant to be `import`ed or not, but you ask for specific deps to be ignored with
-the `--ignore-unused` option, for example:
-`--ignore-unused black mypy`
+We've introduced a `DEFAULT_IGNORE_UNUSED` list, which includes various
+categories of commonly used development tools and dependencies.
+FawltyDeps can automatically ignore these dependencies when checking for unused
+imports. For the complete list, please see the `DEFAULT_IGNORE_UNUSED`
+variable in the [`fawltydeps/settings.py`](./fawltydeps/settings.py) file
+in the repository. If you have additional dependencies that you want to exclude
+from the check for unused imports, you can use the `--ignore-unused` option
+to customize the ignore list. By providing your own list of dependencies with
+this option, you can effectively overwrite the default list. For example:
+`--ignore-unused black mypy some_other_module`
 
 ### Output formats
 
@@ -364,8 +370,12 @@ Here is a complete list of configuration directives we support:
   undeclared dependencies, for example: `["some_module", "some_other_module"]`.
   The default is the empty list: `ignore_undeclared = []`.
 - `ignore_unused`: A list of specific dependencies to ignore when reporting
-  unused dependencies, for example: `["black", "mypy"]`.
-  The default is the empty list: `ignore_unused = []`.
+  unused dependencies, for example: `["black", "mypy", "some_other_module"]`.
+  The default is a list including common development tools. However, you have the
+  flexibility to overwrite this list according to your project's specific requirements.
+  For the complete default list, please see the `DEFAULT_IGNORE_UNUSED`
+  variable in the [`fawltydeps/settings.py`](./fawltydeps/settings.py) file
+  in the repository.
 - `deps_parser_choice`: Manually select which format to use for parsing
   declared dependencies. Must be one of `"requirements.txt"`, `"setup.py"`,
   `"setup.cfg"`, `"pyproject.toml"`, or leave it unset (i.e. the default) for

fawltydeps/cli_parser.py

@@ -232,8 +232,10 @@ def populate_parser_configuration(parser: argparse._ActionsContainer) -> None:
         action="union",
         metavar="DEP_NAME",
         help=(
-            "Dependencies to ignore when looking for unused"
-            " dependencies, e.g. --ignore-unused pylint black"
+            "Specify a list of dependencies to ignore when looking for unused"
+            " dependencies. By default, this list includes common development tools."
+            " Use this option to customize the list,"
+            " e.g. --ignore-unused pylint black some_other_module"
         ),
     )
     parser.add_argument(

fawltydeps/settings.py

@@ -98,6 +98,46 @@ def parse_path_or_stdin(arg: str) -> PathOrSpecial:
     return Path(arg)
 
 
+DEFAULT_IGNORE_UNUSED = {
+    # Development tools not meant to be imported
+    # Formatting Tools
+    "autopep8",
+    "black",
+    "codespell",
+    "isort",
+    "pyformat",
+    "yapf",
+    # Linting Tools
+    "flake8",
+    "mccabe",
+    "mypy",
+    "pyflakes",
+    "pylint",
+    "pyright",
+    "ruff",
+    # Security Tools
+    "bandit",
+    # Documentation Tools
+    "myst-parser",
+    "recommonmark",
+    "sphinx",
+    # Testing Tools
+    "coverage",
+    "fawltydeps",
+    "nox",
+    "pre-commit",
+    "pytest",
+    "tox",
+    # Building and Packaging Tools
+    "twine",
+    "wheel",
+    # Utility Tools
+    "pydocstyle",
+    "rope",
+    "unify",
+}
+
+
 class Settings(BaseSettings):
     """FawltyDeps settings.
 
@@ -119,7 +159,7 @@ class Settings(BaseSettings):
     pyenvs: Set[Path] = {Path(".")}
     custom_mapping: Optional[CustomMapping] = None
     ignore_undeclared: Set[str] = set()
-    ignore_unused: Set[str] = set()
+    ignore_unused: Set[str] = DEFAULT_IGNORE_UNUSED
     deps_parser_choice: Optional[ParserChoice] = None
     install_deps: bool = False
     verbosity: int = 0

pyproject.toml

@@ -74,12 +74,11 @@ types-setuptools = "^65.6.0.2"
 optional = true
 
 [tool.poetry.group.format.dependencies]
-black = "^22"
-colorama = "^0.4.6"
+black = {version = "^22", extras = ["colorama"]}
 isort = [
     # isort 5.12.0 drops support for Python v3.7:
-    {version = "^5.10", python = ">=3.8"},
-    {version = ">=5.10,<5.12.0", python = "<3.8"},
+    {version = "^5.10", extras = ["colors"], python = ">=3.8"},
+    {version = ">=5.10,<5.12.0", extras = ["colors"], python = "<3.8"},
 ]
 codespell = "^2.2.4"
 
@@ -91,9 +90,8 @@ optional = true
 # something to the above groups (i.e. something that targets a specific purpose
 # (e.g. a CI action), consider whether it's also useful to have this available
 # in a developers environment
-black = "^22"
+black = {version = "^22", extras = ["colorama"]}
 codespell = "^2.2.4"
-colorama = "^0.4.6"
 hypothesis = "^6.68.2"
 mypy = "^1.0.1"
 nox = "^2022.11.21"
@@ -157,14 +155,11 @@ build-backend = "poetry.core.masonry.api"
 code = ["fawltydeps"]
 deps = ["pyproject.toml"]
 ignore_unused = [
-    # Tools not meant to be imported
     "black",
     "codespell",
     "hypothesis",
     "mypy",
     "nox",
     "pylint",
     "pytest",
-    # Other dependencies that enable functionality in the above tools
-    "colorama",
 ]

tests/real_projects/requests.toml

@@ -62,6 +62,7 @@ undeclared_deps = [
     "setuptools",
     "simplejson",
 ]
+# We don't have "wheel" as unused_deps because it is in the default_ignore_unused list
 unused_deps =  [
     "Flask",
     "PySocks",
@@ -71,5 +72,4 @@ unused_deps =  [
     "pytest-cov",
     "pytest-httpbin",
     "pytest-mock",
-    "wheel",
 ]

tests/sample_projects/blog_post_example/expected.toml

@@ -8,7 +8,7 @@
 name = "blog_post_example"
 description = '''
   This is the example being used in the blog post that announces FawltyDeps
-  on Tweag's blow: https://www.tweag.io/blog/
+  on Tweag's blog: https://www.tweag.io/blog/
   This example project is contrived to demonstrate what FawltyDeps can do, and
   and what benefits it can bring to a Python project. The project includes:
   - A couple of undeclared dependencies, one that should obviously have been
@@ -42,7 +42,7 @@ undeclared_deps = [
 ]
 
 # Declared dependencies which were never `import`ed from the code:
+# "black" will be ignored since it is in the default_ignored_unused list
 unused_deps = [
-    "black",
     "tensorflow",
 ]

tests/sample_projects/mixed_project/expected.toml

@@ -18,7 +18,8 @@ requirements = []  # rely on identity mapping
 undeclared_deps = ["tomli"]
 
 # Declared dependencies which were never `import`ed from the code:
-unused_deps = ["black", "jieba", "tox"]
+# "black" and "tox" will be ignored since they are in the default_ignored_unused list
+unused_deps = ["jieba"]
 
 [experiments.subdir2]
 description = "Run fawltydeps on subdir2 only"