Integrate warnings filtering directly into Config (#7700)

Warnings are a central part of Python, so much that Python itself has
command-line and environtment variables to handle warnings.

By moving the concept of warning handling into Config, it becomes natural to
filter warnings issued as early as possible, even before the "_pytest.warnings"
plugin is given a chance to spring into action. This also avoids the weird
coupling between config and the warnings plugin that was required before.

Fix #6681
Fix #2891
Fix #7620
Fix #7626
Close #7649

Co-authored-by: Ran Benita <[email protected]>

changelog/6681.improvement.rst

@@ -0,0 +1,3 @@
+Internal pytest warnings issued during the early stages of initialization are now properly handled and can filtered through :confval:`filterwarnings` or ``--pythonwarnings/-W``.
+
+This also fixes a number of long standing issues: `#2891 <https://github.com/pytest-dev/pytest/issues/2891>`__, `#7620 <https://github.com/pytest-dev/pytest/issues/7620>`__, `#7426 <https://github.com/pytest-dev/pytest/issues/7426>`__.

src/_pytest/assertion/rewrite.py

@@ -267,13 +267,11 @@ def mark_rewrite(self, *names: str) -> None:
 
     def _warn_already_imported(self, name: str) -> None:
         from _pytest.warning_types import PytestAssertRewriteWarning
-        from _pytest.warnings import _issue_warning_captured
 
-        _issue_warning_captured(
+        self.config.issue_config_time_warning(
             PytestAssertRewriteWarning(
                 "Module already imported so cannot be rewritten: %s" % name
             ),
-            self.config.hook,
             stacklevel=5,
         )
 

src/_pytest/config/__init__.py

@@ -6,6 +6,7 @@
 import enum
 import inspect
 import os
+import re
 import shlex
 import sys
 import types
@@ -15,6 +16,7 @@
 from typing import Any
 from typing import Callable
 from typing import Dict
+from typing import Generator
 from typing import IO
 from typing import Iterable
 from typing import Iterator
@@ -342,6 +344,13 @@ def __init__(self) -> None:
         self._noconftest = False
         self._duplicatepaths = set()  # type: Set[py.path.local]
 
+        # plugins that were explicitly skipped with pytest.skip
+        # list of (module name, skip reason)
+        # previously we would issue a warning when a plugin was skipped, but
+        # since we refactored warnings as first citizens of Config, they are
+        # just stored here to be used later.
+        self.skipped_plugins = []  # type: List[Tuple[str, str]]
+
         self.add_hookspecs(_pytest.hookspec)
         self.register(self)
         if os.environ.get("PYTEST_DEBUG"):
@@ -694,13 +703,7 @@ def import_plugin(self, modname: str, consider_entry_points: bool = False) -> No
             ).with_traceback(e.__traceback__) from e
 
         except Skipped as e:
-            from _pytest.warnings import _issue_warning_captured
-
-            _issue_warning_captured(
-                PytestConfigWarning("skipped plugin {!r}: {}".format(modname, e.msg)),
-                self.hook,
-                stacklevel=2,
-            )
+            self.skipped_plugins.append((modname, e.msg or ""))
         else:
             mod = sys.modules[importspec]
             self.register(mod, modname)
@@ -1092,6 +1095,9 @@ def _preparse(self, args: List[str], addopts: bool = True) -> None:
                 self._validate_args(self.getini("addopts"), "via addopts config") + args
             )
 
+        self.known_args_namespace = self._parser.parse_known_args(
+            args, namespace=copy.copy(self.option)
+        )
         self._checkversion()
         self._consider_importhook(args)
         self.pluginmanager.consider_preparse(args, exclude_only=False)
@@ -1100,10 +1106,10 @@ def _preparse(self, args: List[str], addopts: bool = True) -> None:
             # plugins are going to be loaded.
             self.pluginmanager.load_setuptools_entrypoints("pytest11")
         self.pluginmanager.consider_env()
-        self.known_args_namespace = ns = self._parser.parse_known_args(
-            args, namespace=copy.copy(self.option)
-        )
+
         self._validate_plugins()
+        self._warn_about_skipped_plugins()
+
         if self.known_args_namespace.confcutdir is None and self.inifile:
             confcutdir = py.path.local(self.inifile).dirname
             self.known_args_namespace.confcutdir = confcutdir
@@ -1112,21 +1118,24 @@ def _preparse(self, args: List[str], addopts: bool = True) -> None:
                 early_config=self, args=args, parser=self._parser
             )
         except ConftestImportFailure as e:
-            if ns.help or ns.version:
+            if self.known_args_namespace.help or self.known_args_namespace.version:
                 # we don't want to prevent --help/--version to work
                 # so just let is pass and print a warning at the end
-                from _pytest.warnings import _issue_warning_captured
-
-                _issue_warning_captured(
+                self.issue_config_time_warning(
                     PytestConfigWarning(
                         "could not load initial conftests: {}".format(e.path)
                     ),
-                    self.hook,
                     stacklevel=2,
                 )
             else:
                 raise
-        self._validate_keys()
+
+    @hookimpl(hookwrapper=True)
+    def pytest_collection(self) -> Generator[None, None, None]:
+        """Validate invalid ini keys after collection is done so we take in account
+        options added by late-loading conftest files."""
+        yield
+        self._validate_config_options()
 
     def _checkversion(self) -> None:
         import pytest
@@ -1147,9 +1156,9 @@ def _checkversion(self) -> None:
                     % (self.inifile, minver, pytest.__version__,)
                 )
 
-    def _validate_keys(self) -> None:
+    def _validate_config_options(self) -> None:
         for key in sorted(self._get_unknown_ini_keys()):
-            self._warn_or_fail_if_strict("Unknown config ini key: {}\n".format(key))
+            self._warn_or_fail_if_strict("Unknown config option: {}\n".format(key))
 
     def _validate_plugins(self) -> None:
         required_plugins = sorted(self.getini("required_plugins"))
@@ -1165,7 +1174,6 @@ def _validate_plugins(self) -> None:
 
         missing_plugins = []
         for required_plugin in required_plugins:
-            spec = None
             try:
                 spec = Requirement(required_plugin)
             except InvalidRequirement:
@@ -1187,11 +1195,7 @@ def _warn_or_fail_if_strict(self, message: str) -> None:
         if self.known_args_namespace.strict_config:
             fail(message, pytrace=False)
 
-        from _pytest.warnings import _issue_warning_captured
-
-        _issue_warning_captured(
-            PytestConfigWarning(message), self.hook, stacklevel=3,
-        )
+        self.issue_config_time_warning(PytestConfigWarning(message), stacklevel=3)
 
     def _get_unknown_ini_keys(self) -> List[str]:
         parser_inicfg = self._parser._inidict
@@ -1222,6 +1226,49 @@ def parse(self, args: List[str], addopts: bool = True) -> None:
         except PrintHelp:
             pass
 
+    def issue_config_time_warning(self, warning: Warning, stacklevel: int) -> None:
+        """Issue and handle a warning during the "configure" stage.
+
+        During ``pytest_configure`` we can't capture warnings using the ``catch_warnings_for_item``
+        function because it is not possible to have hookwrappers around ``pytest_configure``.
+
+        This function is mainly intended for plugins that need to issue warnings during
+        ``pytest_configure`` (or similar stages).
+
+        :param warning: The warning instance.
+        :param stacklevel: stacklevel forwarded to warnings.warn.
+        """
+        if self.pluginmanager.is_blocked("warnings"):
+            return
+
+        cmdline_filters = self.known_args_namespace.pythonwarnings or []
+        config_filters = self.getini("filterwarnings")
+
+        with warnings.catch_warnings(record=True) as records:
+            warnings.simplefilter("always", type(warning))
+            apply_warning_filters(config_filters, cmdline_filters)
+            warnings.warn(warning, stacklevel=stacklevel)
+
+        if records:
+            frame = sys._getframe(stacklevel - 1)
+            location = frame.f_code.co_filename, frame.f_lineno, frame.f_code.co_name
+            self.hook.pytest_warning_captured.call_historic(
+                kwargs=dict(
+                    warning_message=records[0],
+                    when="config",
+                    item=None,
+                    location=location,
+                )
+            )
+            self.hook.pytest_warning_recorded.call_historic(
+                kwargs=dict(
+                    warning_message=records[0],
+                    when="config",
+                    nodeid="",
+                    location=location,
+                )
+            )
+
     def addinivalue_line(self, name: str, line: str) -> None:
         """Add a line to an ini-file option. The option must have been
         declared but might not yet be set in which case the line becomes
@@ -1365,8 +1412,6 @@ def getvalueorskip(self, name: str, path=None):
 
     def _warn_about_missing_assertion(self, mode: str) -> None:
         if not _assertion_supported():
-            from _pytest.warnings import _issue_warning_captured
-
             if mode == "plain":
                 warning_text = (
                     "ASSERTIONS ARE NOT EXECUTED"
@@ -1381,8 +1426,15 @@ def _warn_about_missing_assertion(self, mode: str) -> None:
                     "by the underlying Python interpreter "
                     "(are you using python -O?)\n"
                 )
-            _issue_warning_captured(
-                PytestConfigWarning(warning_text), self.hook, stacklevel=3,
+            self.issue_config_time_warning(
+                PytestConfigWarning(warning_text), stacklevel=3,
+            )
+
+    def _warn_about_skipped_plugins(self) -> None:
+        for module_name, msg in self.pluginmanager.skipped_plugins:
+            self.issue_config_time_warning(
+                PytestConfigWarning("skipped plugin {!r}: {}".format(module_name, msg)),
+                stacklevel=2,
             )
 
 
@@ -1435,3 +1487,51 @@ def _strtobool(val: str) -> bool:
         return False
     else:
         raise ValueError("invalid truth value {!r}".format(val))
+
+
+@lru_cache(maxsize=50)
+def parse_warning_filter(
+    arg: str, *, escape: bool
+) -> "Tuple[str, str, Type[Warning], str, int]":
+    """Parse a warnings filter string.
+
+    This is copied from warnings._setoption, but does not apply the filter,
+    only parses it, and makes the escaping optional.
+    """
+    parts = arg.split(":")
+    if len(parts) > 5:
+        raise warnings._OptionError("too many fields (max 5): {!r}".format(arg))
+    while len(parts) < 5:
+        parts.append("")
+    action_, message, category_, module, lineno_ = [s.strip() for s in parts]
+    action = warnings._getaction(action_)  # type: str # type: ignore[attr-defined]
+    category = warnings._getcategory(
+        category_
+    )  # type: Type[Warning] # type: ignore[attr-defined]
+    if message and escape:
+        message = re.escape(message)
+    if module and escape:
+        module = re.escape(module) + r"\Z"
+    if lineno_:
+        try:
+            lineno = int(lineno_)
+            if lineno < 0:
+                raise ValueError
+        except (ValueError, OverflowError) as e:
+            raise warnings._OptionError("invalid lineno {!r}".format(lineno_)) from e
+    else:
+        lineno = 0
+    return action, message, category, module, lineno
+
+
+def apply_warning_filters(
+    config_filters: Iterable[str], cmdline_filters: Iterable[str]
+) -> None:
+    """Applies pytest-configured filters to the warnings module"""
+    # Filters should have this precedence: cmdline options, config.
+    # Filters should be applied in the inverse order of precedence.
+    for arg in config_filters:
+        warnings.filterwarnings(*parse_warning_filter(arg, escape=False))
+
+    for arg in cmdline_filters:
+        warnings.filterwarnings(*parse_warning_filter(arg, escape=True))

src/_pytest/faulthandler.py

@@ -30,18 +30,15 @@ def pytest_configure(config: Config) -> None:
         # of enabling faulthandler before each test executes.
         config.pluginmanager.register(FaultHandlerHooks(), "faulthandler-hooks")
     else:
-        from _pytest.warnings import _issue_warning_captured
-
         # Do not handle dumping to stderr if faulthandler is already enabled, so warn
         # users that the option is being ignored.
         timeout = FaultHandlerHooks.get_timeout_config_value(config)
         if timeout > 0:
-            _issue_warning_captured(
+            config.issue_config_time_warning(
                 pytest.PytestConfigWarning(
                     "faulthandler module enabled before pytest configuration step, "
                     "'faulthandler_timeout' option ignored"
                 ),
-                config.hook,
                 stacklevel=2,
             )
 

src/_pytest/main.py

@@ -69,6 +69,20 @@ def pytest_addoption(parser: Parser) -> None:
         const=1,
         help="exit instantly on first error or failed test.",
     )
+    group = parser.getgroup("pytest-warnings")
+    group.addoption(
+        "-W",
+        "--pythonwarnings",
+        action="append",
+        help="set which warnings to report, see -W option of python itself.",
+    )
+    parser.addini(
+        "filterwarnings",
+        type="linelist",
+        help="Each line specifies a pattern for "
+        "warnings.filterwarnings. "
+        "Processed after -W/--pythonwarnings.",
+    )
     group._addoption(
         "--maxfail",
         metavar="num",