mypy --strict for sydent.http.matrixfederationagent (#444)

* Let's typecheck matrix federation agent

* Improve type stub for twisted.python.log.err

* Additional stubs that apply to matrixfederationagent

* Annotate the _RoutingResult struct

* Remove unused typeignore (now in mypy config)

* Annotate well_known_cache

* Annotate _parse_cache_control

* Annotate _cache_period_from_headers

* Annotate LoggingHostnameEndpoint

* Annotate _do_get_well_known

* Workaround no annotation for Headers.copy

* annotate EndpointFactory

* Avoid str/bytes confusion in well_known handling

* Annotations for MatrixFederationAgent

* Suppress reactor complaint for now

* sydent.http/*.py now passes mypy --strict

* Isort

* Changelog

* Additional linting --- looks like it didn't fully run?

* Keep 3.6 flake8 happy with annotations on previous line

* Review fixup

changelog.d/444.misc

@@ -0,0 +1 @@
+Get `sydent.http.matrixfederationagent` to pass `mypy --strict`.

pyproject.toml

@@ -49,16 +49,11 @@ strict = true
 files = [
     # Find files that pass with
     #     find sydent tests -type d -not -name __pycache__ -exec bash -c "mypy --strict '{}' > /dev/null"  \; -print
+    # TODO "sydent/*.py"
     "sydent/config",
     "sydent/db",
-    "sydent/http/auth.py",
-    "sydent/http/blacklisting_reactor.py",
-    "sydent/http/federation_tls_options.py",
-    "sydent/http/httpclient.py",
-    "sydent/http/httpcommon.py",
-    "sydent/http/httpsclient.py",
-    "sydent/http/httpserver.py",
-    "sydent/http/srvresolver.py",
+    "sydent/http/*.py",
+    # TODO "sydent/http/servlets",
     "sydent/hs_federation",
     "sydent/replication",
     "sydent/sms",

stubs/twisted/internet/endpoints.pyi

@@ -0,0 +1,32 @@
+from typing import Any, AnyStr, Optional
+
+from twisted.internet import interfaces
+from twisted.internet.defer import Deferred
+from twisted.internet.interfaces import (
+    IOpenSSLClientConnectionCreator,
+    IProtocol,
+    IProtocolFactory,
+    IStreamClientEndpoint,
+)
+from zope.interface import implementer
+
+@implementer(interfaces.IStreamClientEndpoint)
+class HostnameEndpoint:
+    # Reactor should be a "provider of L{IReactorTCP}, L{IReactorTime} and
+    # either L{IReactorPluggableNameResolver} or L{IReactorPluggableResolver}."
+    # I don't know how to encode that in the type system.
+    def __init__(
+        self,
+        reactor: object,
+        host: AnyStr,
+        port: int,
+        timeout: float = 30,
+        bindAddress: Optional[bytes] = None,
+        attemptDelay: Optional[float] = None,
+    ): ...
+    def connect(self, protocol_factory: IProtocolFactory) -> Deferred[IProtocol]: ...
+
+def wrapClientTLS(
+    connectionCreator: IOpenSSLClientConnectionCreator,
+    wrappedEndpoint: IStreamClientEndpoint,
+) -> IStreamClientEndpoint: ...

stubs/twisted/python/log.pyi

@@ -5,5 +5,5 @@ from twisted.python.failure import Failure
 def err(
     _stuff: Union[None, Exception, Failure] = None,
     _why: Optional[str] = None,
-    **kw: Any,
+    **kw: object,
 ) -> None: ...

stubs/twisted/web/client.pyi

@@ -9,7 +9,13 @@ from twisted.internet.interfaces import (
 )
 from twisted.internet.task import Cooperator
 from twisted.web.http_headers import Headers
-from twisted.web.iweb import IAgent, IBodyProducer, IPolicyForHTTPS, IResponse
+from twisted.web.iweb import (
+    IAgent,
+    IAgentEndpointFactory,
+    IBodyProducer,
+    IPolicyForHTTPS,
+    IResponse,
+)
 from zope.interface import implementer
 
 C = TypeVar("C")
@@ -20,13 +26,23 @@ class BrowserLikePolicyForHTTPS:
         self, hostname: bytes, port: int
     ) -> IOpenSSLClientConnectionCreator: ...
 
-class HTTPConnectionPool: ...
+class HTTPConnectionPool:
+    persistent: bool
+    maxPersistentPerHost: int
+    cachedConnectionTimeout: float
+    retryAutomatically: bool
+    def __init__(self, reactor: object, persistent: bool = True): ...
 
 @implementer(IAgent)
 class Agent:
+    # Here and in `usingEndpointFactory`, reactor should be a "provider of
+    # L{IReactorTCP}, L{IReactorTime} and either
+    # L{IReactorPluggableNameResolver} or L{IReactorPluggableResolver}."
+    # I don't know how to encode that in the type system; see also
+    # https://github.com/Shoobx/mypy-zope/issues/58
     def __init__(
         self,
-        reactor: Any,
+        reactor: object,
         contextFactory: IPolicyForHTTPS = BrowserLikePolicyForHTTPS(),
         connectTimeout: Optional[float] = None,
         bindAddress: Optional[bytes] = None,
@@ -39,17 +55,20 @@ class Agent:
         headers: Optional[Headers] = None,
         bodyProducer: Optional[IBodyProducer] = None,
     ) -> Deferred[IResponse]: ...
+    @classmethod
+    def usingEndpointFactory(
+        cls: Type[C],
+        reactor: object,
+        endpointFactory: IAgentEndpointFactory,
+        pool: Optional[HTTPConnectionPool] = None,
+    ) -> C: ...
 
 @implementer(IBodyProducer)
 class FileBodyProducer:
     def __init__(
         self,
         inputFile: BinaryIO,
-        # Type safety: twisted.internet.task.cooperate is a function with the
-        # same signature as Cooperator.cooperate. (It just wraps a module-level
-        # global cooperator.) But there's no easy way to annotate "either this
-        # type or a specific module".
-        cooperator: Cooperator = twisted.internet.task,  # type: ignore[assignment]
+        cooperator: Cooperator = ...,
         readSize: int = 2 ** 16,
     ): ...
     # Length is either `int` or the opaque object UNKNOWN_LENGTH.
@@ -95,3 +114,14 @@ class URI:
     ): ...
     @classmethod
     def fromBytes(cls: Type[C], uri: bytes, defaultPort: Optional[int] = None) -> C: ...
+
+@implementer(IAgent)
+class RedirectAgent:
+    def __init__(self, Agent: Agent, redirectLimit: int = 20): ...
+    def request(
+        self,
+        method: bytes,
+        uri: bytes,
+        headers: Optional[Headers] = None,
+        bodyProducer: Optional[IBodyProducer] = None,
+    ) -> Deferred[IResponse]: ...

stubs/twisted/web/http.pyi

@@ -51,3 +51,5 @@ class Request:
 class PotentialDataLoss(Exception): ...
 
 CACHED: object
+
+def stringToDatetime(dateString: bytes) -> int: ...

sydent/http/httpclient.py

@@ -177,7 +177,12 @@ class FederationHttpClient(HTTPClient[MatrixFederationAgent]):
     def __init__(self, sydent: "Sydent") -> None:
         self.sydent = sydent
         self.agent = MatrixFederationAgent(
-            BlacklistingReactorWrapper(
+            # Type-safety: I don't have a good way of expressing that
+            # the reactor is IReactorTCP, IReactorTime and
+            # IReactorPluggableNameResolver all at once. But it is, because
+            # it wraps the sydent reactor.
+            # TODO: can we introduce a SydentReactor type like SynapseReactor?
+            BlacklistingReactorWrapper(  # type: ignore[arg-type]
                 reactor=self.sydent.reactor,
                 ip_whitelist=sydent.config.general.ip_whitelist,
                 ip_blacklist=sydent.config.general.ip_blacklist,