Merge pull request #95 from scrapy/pyupgrade

Add pyupgrade, bump some tools

.github/workflows/main.yml

@@ -70,4 +70,4 @@ jobs:
         env: ${{ matrix.env }}
         run: tox
       - name: Publish coverage data
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v5

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ repos:
       - id: bandit
         args: [-r, -c, .bandit.yml]
     repo: https://github.com/PyCQA/bandit
-    rev: 1.7.10
+    rev: 1.8.2
   - hooks:
       - id: black
         language_version: python3
@@ -27,3 +27,8 @@ repos:
           - flake8-string-format
     repo: https://github.com/pycqa/flake8
     rev: 7.1.1
+  - hooks:
+      - id: pyupgrade
+        args: [--py39-plus]
+    repo: https://github.com/asottile/pyupgrade
+    rev: v3.19.1

docs/conf.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 #
 # Scrapy documentation build configuration file, created by
 # sphinx-quickstart on Mon Nov 24 12:02:52 2008.

docs/release-notes.rst

@@ -103,9 +103,9 @@ itemloaders 1.0.5 (2022-08-25)
 itemloaders 1.0.4 (2020-11-12)
 ------------------------------
 
--   When adding a :class:`scrapy.item.scrapy.Item` object as a value into an
+-   When adding a :class:`scrapy.Item` object as a value into an
     :class:`ItemLoader` object, that item is now added *as is*, instead of
-    becoming a :class:`list` of keys from its :attr:`scrapy.item.scrapy.Item.fields`
+    becoming a :class:`list` of keys from its :attr:`scrapy.Item.fields`
     (:gh:`28`, :gh:`29`)
 
 -   Increased test coverage (:gh:`27`)

itemloaders/__init__.py

@@ -6,19 +6,10 @@
 
 from __future__ import annotations
 
+from collections.abc import Iterable, MutableMapping
 from contextlib import suppress
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    Callable,
-    Dict,
-    Iterable,
-    List,
-    MutableMapping,
-    Optional,
-    Pattern,
-    Union,
-)
+from re import Pattern
+from typing import TYPE_CHECKING, Any, Callable
 
 from itemadapter import ItemAdapter
 from parsel import Selector
@@ -122,26 +113,26 @@ class Product:
     def __init__(
         self,
         item: Any = None,
-        selector: Optional[Selector] = None,
-        parent: Optional[ItemLoader] = None,
+        selector: Selector | None = None,
+        parent: ItemLoader | None = None,
         **context: Any,
     ):
-        self.selector: Optional[Selector] = selector
+        self.selector: Selector | None = selector
         context.update(selector=selector)
         if item is None:
             item = self.default_item_class()
         self._local_item = item
         context["item"] = item
         self.context: MutableMapping[str, Any] = context
-        self.parent: Optional[ItemLoader] = parent
-        self._local_values: Dict[str, List[Any]] = {}
+        self.parent: ItemLoader | None = parent
+        self._local_values: dict[str, list[Any]] = {}
         # values from initial item
         for field_name, value in ItemAdapter(item).items():
             self._values.setdefault(field_name, [])
             self._values[field_name] += arg_to_iter(value)
 
     @property
-    def _values(self) -> Dict[str, List[Any]]:
+    def _values(self) -> dict[str, list[Any]]:
         if self.parent is not None:
             return self.parent._values
         else:
@@ -186,10 +177,10 @@ def nested_css(self, css: str, **context: Any) -> Self:
 
     def add_value(
         self,
-        field_name: Optional[str],
+        field_name: str | None,
         value: Any,
         *processors: Callable[..., Any],
-        re: Union[str, Pattern[str], None] = None,
+        re: str | Pattern[str] | None = None,
         **kw: Any,
     ) -> Self:
         """
@@ -229,10 +220,10 @@ def add_value(
 
     def replace_value(
         self,
-        field_name: Optional[str],
+        field_name: str | None,
         value: Any,
         *processors: Callable[..., Any],
-        re: Union[str, Pattern[str], None] = None,
+        re: str | Pattern[str] | None = None,
         **kw: Any,
     ) -> Self:
         """
@@ -267,7 +258,7 @@ def get_value(
         self,
         value: Any,
         *processors: Callable[..., Any],
-        re: Union[str, Pattern[str], None] = None,
+        re: str | Pattern[str] | None = None,
         **kw: Any,
     ) -> Any:
         """
@@ -337,7 +328,7 @@ def get_output_value(self, field_name: str) -> Any:
                 % (field_name, value, type(e).__name__, str(e))
             ) from e
 
-    def get_collected_values(self, field_name: str) -> List[Any]:
+    def get_collected_values(self, field_name: str) -> list[Any]:
         """Return the collected values for the given field."""
         return self._values.get(field_name, [])
 
@@ -391,10 +382,10 @@ def _check_selector_method(self) -> None:
 
     def add_xpath(
         self,
-        field_name: Optional[str],
-        xpath: Union[str, Iterable[str]],
+        field_name: str | None,
+        xpath: str | Iterable[str],
         *processors: Callable[..., Any],
-        re: Union[str, Pattern[str], None] = None,
+        re: str | Pattern[str] | None = None,
         **kw: Any,
     ) -> Self:
         """
@@ -423,10 +414,10 @@ def add_xpath(
 
     def replace_xpath(
         self,
-        field_name: Optional[str],
-        xpath: Union[str, Iterable[str]],
+        field_name: str | None,
+        xpath: str | Iterable[str],
         *processors: Callable[..., Any],
-        re: Union[str, Pattern[str], None] = None,
+        re: str | Pattern[str] | None = None,
         **kw: Any,
     ) -> Self:
         """
@@ -441,9 +432,9 @@ def replace_xpath(
 
     def get_xpath(
         self,
-        xpath: Union[str, Iterable[str]],
+        xpath: str | Iterable[str],
         *processors: Callable[..., Any],
-        re: Union[str, Pattern[str], None] = None,
+        re: str | Pattern[str] | None = None,
         **kw: Any,
     ) -> Any:
         """
@@ -469,20 +460,18 @@ def get_xpath(
         values = self._get_xpathvalues(xpath, **kw)
         return self.get_value(values, *processors, re=re, **kw)
 
-    def _get_xpathvalues(
-        self, xpaths: Union[str, Iterable[str]], **kw: Any
-    ) -> List[Any]:
+    def _get_xpathvalues(self, xpaths: str | Iterable[str], **kw: Any) -> list[Any]:
         self._check_selector_method()
         assert self.selector is not None
         xpaths = arg_to_iter(xpaths)
         return flatten(self.selector.xpath(xpath, **kw).getall() for xpath in xpaths)
 
     def add_css(
         self,
-        field_name: Optional[str],
-        css: Union[str, Iterable[str]],
+        field_name: str | None,
+        css: str | Iterable[str],
         *processors: Callable[..., Any],
-        re: Union[str, Pattern[str], None] = None,
+        re: str | Pattern[str] | None = None,
         **kw: Any,
     ) -> Self:
         """
@@ -511,10 +500,10 @@ def add_css(
 
     def replace_css(
         self,
-        field_name: Optional[str],
-        css: Union[str, Iterable[str]],
+        field_name: str | None,
+        css: str | Iterable[str],
         *processors: Callable[..., Any],
-        re: Union[str, Pattern[str], None] = None,
+        re: str | Pattern[str] | None = None,
         **kw: Any,
     ) -> Self:
         """
@@ -529,9 +518,9 @@ def replace_css(
 
     def get_css(
         self,
-        css: Union[str, Iterable[str]],
+        css: str | Iterable[str],
         *processors: Callable[..., Any],
-        re: Union[str, Pattern[str], None] = None,
+        re: str | Pattern[str] | None = None,
         **kw: Any,
     ) -> Any:
         """
@@ -556,18 +545,18 @@ def get_css(
         values = self._get_cssvalues(css)
         return self.get_value(values, *processors, re=re, **kw)
 
-    def _get_cssvalues(self, csss: Union[str, Iterable[str]]) -> List[Any]:
+    def _get_cssvalues(self, csss: str | Iterable[str]) -> list[Any]:
         self._check_selector_method()
         assert self.selector is not None
         csss = arg_to_iter(csss)
         return flatten(self.selector.css(css).getall() for css in csss)
 
     def add_jmes(
         self,
-        field_name: Optional[str],
+        field_name: str | None,
         jmes: str,
         *processors: Callable[..., Any],
-        re: Union[str, Pattern[str], None] = None,
+        re: str | Pattern[str] | None = None,
         **kw: Any,
     ) -> Self:
         """
@@ -595,10 +584,10 @@ def add_jmes(
 
     def replace_jmes(
         self,
-        field_name: Optional[str],
-        jmes: Union[str, Iterable[str]],
+        field_name: str | None,
+        jmes: str | Iterable[str],
         *processors: Callable[..., Any],
-        re: Union[str, Pattern[str], None] = None,
+        re: str | Pattern[str] | None = None,
         **kw: Any,
     ) -> Self:
         """
@@ -612,9 +601,9 @@ def replace_jmes(
 
     def get_jmes(
         self,
-        jmes: Union[str, Iterable[str]],
+        jmes: str | Iterable[str],
         *processors: Callable[..., Any],
-        re: Union[str, Pattern[str], None] = None,
+        re: str | Pattern[str] | None = None,
         **kw: Any,
     ) -> Any:
         """
@@ -639,7 +628,7 @@ def get_jmes(
         values = self._get_jmesvalues(jmes)
         return self.get_value(values, *processors, re=re, **kw)
 
-    def _get_jmesvalues(self, jmess: Union[str, Iterable[str]]) -> List[Any]:
+    def _get_jmesvalues(self, jmess: str | Iterable[str]) -> list[Any]:
         self._check_selector_method()
         assert self.selector is not None
         jmess = arg_to_iter(jmess)

itemloaders/common.py

@@ -1,7 +1,10 @@
 """Common functions used in Item Loaders code"""
 
+from __future__ import annotations
+
+from collections.abc import MutableMapping
 from functools import partial
-from typing import Any, Callable, MutableMapping
+from typing import Any, Callable
 
 from itemloaders.utils import get_func_args
 

itemloaders/processors.py

@@ -4,8 +4,11 @@
 See documentation in docs/topics/loaders.rst
 """
 
+from __future__ import annotations
+
 from collections import ChainMap
-from typing import Any, Callable, Iterable, List, MutableMapping, Optional
+from collections.abc import Iterable, MutableMapping
+from typing import Any, Callable
 
 from itemloaders.common import wrap_loader_context
 from itemloaders.utils import arg_to_iter
@@ -60,7 +63,7 @@ def __init__(self, *functions: Callable[..., Any], **default_loader_context: Any
         self.default_loader_context = default_loader_context
 
     def __call__(
-        self, value: Any, loader_context: Optional[MutableMapping[str, Any]] = None
+        self, value: Any, loader_context: MutableMapping[str, Any] | None = None
     ) -> Iterable[Any]:
         values = arg_to_iter(value)
         context: MutableMapping[str, Any]
@@ -70,7 +73,7 @@ def __call__(
             context = self.default_loader_context
         wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions]
         for func in wrapped_funcs:
-            next_values: List[Any] = []
+            next_values: list[Any] = []
             for v in values:
                 try:
                     next_values += arg_to_iter(func(v))
@@ -119,7 +122,7 @@ def __init__(self, *functions: Callable[..., Any], **default_loader_context: Any
         self.default_loader_context = default_loader_context
 
     def __call__(
-        self, value: Any, loader_context: Optional[MutableMapping[str, Any]] = None
+        self, value: Any, loader_context: MutableMapping[str, Any] | None = None
     ) -> Any:
         context: MutableMapping[str, Any]
         if loader_context:

itemloaders/utils.py

@@ -3,9 +3,12 @@
 Refactoring to come later
 """
 
+from __future__ import annotations
+
 import inspect
+from collections.abc import Generator, Iterable
 from functools import partial
-from typing import Any, Callable, Generator, Iterable, List
+from typing import Any, Callable
 
 
 def arg_to_iter(arg: Any) -> Iterable[Any]:
@@ -25,12 +28,12 @@ def arg_to_iter(arg: Any) -> Iterable[Any]:
     return [arg]
 
 
-def get_func_args(func: Callable[..., Any], stripself: bool = False) -> List[str]:
+def get_func_args(func: Callable[..., Any], stripself: bool = False) -> list[str]:
     """Return the argument name list of a callable object"""
     if not callable(func):
         raise TypeError(f"func must be callable, got {type(func).__name__!r}")
 
-    args: List[str] = []
+    args: list[str] = []
     try:
         sig = inspect.signature(func)
     except ValueError:

pylintrc

@@ -3,8 +3,8 @@ ignore=typing
 persistent=no
 
 [MESSAGES CONTROL]
+enable=useless-suppression
 disable=broad-exception-caught,
-        c-extension-no-member,
         consider-using-f-string,
         disallowed-name,
         duplicate-code,
@@ -17,9 +17,7 @@ disable=broad-exception-caught,
         missing-function-docstring,
         missing-module-docstring,
         no-else-return,
-        no-member,
         not-callable,
-        parse-error,
         protected-access,
         raise-missing-from,
         redefined-builtin,
@@ -29,8 +27,4 @@ disable=broad-exception-caught,
         too-many-lines,
         too-many-positional-arguments,
         too-many-public-methods,
-        unidiomatic-typecheck,
         unused-argument,
-        use-a-generator,
-        wrong-import-order,
-        wrong-import-position,