From 9a16f00320b95a5b43eb122ce81761fcd51079b4 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 12:11:24 +0100
Subject: [PATCH 01/21] Let's typecheck matrix federation agent

---
 pyproject.toml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pyproject.toml b/pyproject.toml
index 119f97ff..f9b02b08 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -58,6 +58,7 @@ files = [
     "sydent/http/httpcommon.py",
     "sydent/http/httpsclient.py",
     "sydent/http/httpserver.py",
+    "sydent/http/matrixfederationagent.py",
     "sydent/http/srvresolver.py",
     "sydent/hs_federation",
     "sydent/replication",

From c710fee9469c302f11a34e87ebe736b45a843eba Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 12:14:10 +0100
Subject: [PATCH 02/21] Improve type stub for twisted.python.log.err

---
 stubs/twisted/python/log.pyi | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stubs/twisted/python/log.pyi b/stubs/twisted/python/log.pyi
index 9d9fa686..c719b2c2 100644
--- a/stubs/twisted/python/log.pyi
+++ b/stubs/twisted/python/log.pyi
@@ -5,5 +5,5 @@ from twisted.python.failure import Failure
 def err(
     _stuff: Union[None, Exception, Failure] = None,
     _why: Optional[str] = None,
-    **kw: Any,
+    **kw: object,
 ) -> None: ...

From 18b3c2287614766d0ab936e1b2b513b0f588e15f Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 12:20:30 +0100
Subject: [PATCH 03/21] Additional stubs that apply to matrixfederationagent

---
 stubs/twisted/internet/endpoints.pyi | 32 +++++++++++++++++++
 stubs/twisted/web/client.pyi         | 46 +++++++++++++++++++++++-----
 stubs/twisted/web/http.pyi           |  2 ++
 3 files changed, 72 insertions(+), 8 deletions(-)
 create mode 100644 stubs/twisted/internet/endpoints.pyi

diff --git a/stubs/twisted/internet/endpoints.pyi b/stubs/twisted/internet/endpoints.pyi
new file mode 100644
index 00000000..f4be1022
--- /dev/null
+++ b/stubs/twisted/internet/endpoints.pyi
@@ -0,0 +1,32 @@
+from typing import Optional, AnyStr, Any
+
+from twisted.internet import interfaces
+from twisted.internet.defer import Deferred
+from twisted.internet.interfaces import (
+    IProtocolFactory,
+    IProtocol,
+    IOpenSSLClientConnectionCreator,
+    IStreamClientEndpoint,
+)
+from zope.interface import implementer
+
+@implementer(interfaces.IStreamClientEndpoint)
+class HostnameEndpoint:
+    # Reactor should be a "provider of L{IReactorTCP}, L{IReactorTime} and
+    # either L{IReactorPluggableNameResolver} or L{IReactorPluggableResolver}."
+    # I don't know how to encode that in the type system.
+    def __init__(
+        self,
+        reactor: object,
+        host: AnyStr,
+        port: int,
+        timeout: float = 30,
+        bindAddress: Optional[bytes] = None,
+        attemptDelay: Optional[float] = None,
+    ): ...
+    def connect(self, protocol_factory: IProtocolFactory) -> Deferred[IProtocol]: ...
+
+def wrapClientTLS(
+    connectionCreator: IOpenSSLClientConnectionCreator,
+    wrappedEndpoint: IStreamClientEndpoint,
+) -> IStreamClientEndpoint: ...
diff --git a/stubs/twisted/web/client.pyi b/stubs/twisted/web/client.pyi
index e46f994c..2e1a5515 100644
--- a/stubs/twisted/web/client.pyi
+++ b/stubs/twisted/web/client.pyi
@@ -9,7 +9,13 @@ from twisted.internet.interfaces import (
 )
 from twisted.internet.task import Cooperator
 from twisted.web.http_headers import Headers
-from twisted.web.iweb import IAgent, IBodyProducer, IPolicyForHTTPS, IResponse
+from twisted.web.iweb import (
+    IAgent,
+    IBodyProducer,
+    IPolicyForHTTPS,
+    IResponse,
+    IAgentEndpointFactory,
+)
 from zope.interface import implementer
 
 C = TypeVar("C")
@@ -20,13 +26,23 @@ class BrowserLikePolicyForHTTPS:
         self, hostname: bytes, port: int
     ) -> IOpenSSLClientConnectionCreator: ...
 
-class HTTPConnectionPool: ...
+class HTTPConnectionPool:
+    persistent: bool
+    maxPersistentPerHost: int
+    cachedConnectionTimeout: int  # (maybe float?)
+    retryAutomatically: bool
+    def __init__(self, reactor: object, persistent: bool = True): ...
 
 @implementer(IAgent)
 class Agent:
+    # Here and in `usingEndpointFactory`, reactor should be a "provider of
+    # L{IReactorTCP}, L{IReactorTime} and either
+    # L{IReactorPluggableNameResolver} or L{IReactorPluggableResolver}."
+    # I don't know how to encode that in the type system; see also
+    # https://github.com/Shoobx/mypy-zope/issues/58
     def __init__(
         self,
-        reactor: Any,
+        reactor: object,
         contextFactory: IPolicyForHTTPS = BrowserLikePolicyForHTTPS(),
         connectTimeout: Optional[float] = None,
         bindAddress: Optional[bytes] = None,
@@ -39,17 +55,20 @@ class Agent:
         headers: Optional[Headers] = None,
         bodyProducer: Optional[IBodyProducer] = None,
     ) -> Deferred[IResponse]: ...
+    @classmethod
+    def usingEndpointFactory(
+        cls: Type[C],
+        reactor: object,
+        endpointFactory: IAgentEndpointFactory,
+        pool: Optional[HTTPConnectionPool] = None,
+    ) -> C: ...
 
 @implementer(IBodyProducer)
 class FileBodyProducer:
     def __init__(
         self,
         inputFile: BinaryIO,
-        # Type safety: twisted.internet.task.cooperate is a function with the
-        # same signature as Cooperator.cooperate. (It just wraps a module-level
-        # global cooperator.) But there's no easy way to annotate "either this
-        # type or a specific module".
-        cooperator: Cooperator = twisted.internet.task,  # type: ignore[assignment]
+        cooperator: Cooperator = ...,
         readSize: int = 2 ** 16,
     ): ...
     # Length is either `int` or the opaque object UNKNOWN_LENGTH.
@@ -95,3 +114,14 @@ class URI:
     ): ...
     @classmethod
     def fromBytes(cls: Type[C], uri: bytes, defaultPort: Optional[int] = None) -> C: ...
+
+@implementer(IAgent)
+class RedirectAgent:
+    def __init__(self, Agent: Agent, redirectLimit: int = 20): ...
+    def request(
+        self,
+        method: bytes,
+        uri: bytes,
+        headers: Optional[Headers] = None,
+        bodyProducer: Optional[IBodyProducer] = None,
+    ) -> Deferred[IResponse]: ...
diff --git a/stubs/twisted/web/http.pyi b/stubs/twisted/web/http.pyi
index 7d809836..487ef069 100644
--- a/stubs/twisted/web/http.pyi
+++ b/stubs/twisted/web/http.pyi
@@ -51,3 +51,5 @@ class Request:
 class PotentialDataLoss(Exception): ...
 
 CACHED: object
+
+def stringToDatetime(dateString: bytes) -> int: ...

From b56dcd5145ca742993ed0a7d7a401fb1e54ba982 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 12:26:00 +0100
Subject: [PATCH 04/21] Annotate the _RoutingResult struct

---
 sydent/http/matrixfederationagent.py | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/sydent/http/matrixfederationagent.py b/sydent/http/matrixfederationagent.py
index ff07f7e7..474c6654 100644
--- a/sydent/http/matrixfederationagent.py
+++ b/sydent/http/matrixfederationagent.py
@@ -141,7 +141,9 @@ def request(
             (including problems that prevent the request from being sent).
         """
         parsed_uri = URI.fromBytes(uri, defaultPort=-1)
-        res = yield defer.ensureDeferred(self._route_matrix_uri(parsed_uri))
+        routing: _RoutingResult = yield defer.ensureDeferred(
+            self._route_matrix_uri(parsed_uri)
+        )
 
         # set up the TLS connection params
         #
@@ -152,7 +154,7 @@ def request(
             tls_options = None
         else:
             tls_options = self._tls_client_options_factory.get_options(
-                res.tls_server_name.decode("ascii")
+                routing.tls_server_name.decode("ascii")
             )
 
         # make sure that the Host header is set correctly
@@ -163,15 +165,15 @@ def request(
             assert headers is not None
 
         if not headers.hasHeader(b"host"):
-            headers.addRawHeader(b"host", res.host_header)
+            headers.addRawHeader(b"host", routing.host_header)
 
         class EndpointFactory:
             @staticmethod
             def endpointForURI(_uri):
                 ep = LoggingHostnameEndpoint(
                     self._reactor,
-                    res.target_host,
-                    res.target_port,
+                    routing.target_host,
+                    routing.target_port,
                 )
                 if tls_options is not None:
                     ep = wrapClientTLS(tls_options, ep)
@@ -412,7 +414,7 @@ def _parse_cache_control(headers):
     return cache_controls
 
 
-@attr.s
+@attr.s(frozen=True, slots=True, auto_attribs=True)
 class _RoutingResult:
     """The result returned by `_route_matrix_uri`.
     Contains the parameters needed to direct a federation connection to a particular
@@ -421,28 +423,28 @@ class _RoutingResult:
     chosen from the list.
     """
 
-    host_header = attr.ib()
+    host_header: bytes
     """
     The value we should assign to the Host header (host:port from the matrix
     URI, or .well-known).
     :type: bytes
     """
 
-    tls_server_name = attr.ib()
+    tls_server_name: bytes
     """
     The server name we should set in the SNI (typically host, without port, from the
     matrix URI or .well-known)
     :type: bytes
     """
 
-    target_host = attr.ib()
+    target_host: bytes
     """
     The hostname (or IP literal) we should route the TCP connection to (the target of the
     SRV record, or the hostname from the URL/.well-known)
     :type: bytes
     """
 
-    target_port = attr.ib()
+    target_port: int
     """
     The port we should route the TCP connection to (the target of the SRV record, or
     the port from the URL/.well-known, or 8448)

From e0b39ee82b7126eec9576b34a43c7565500b4d6b Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 12:26:52 +0100
Subject: [PATCH 05/21] Remove unused typeignore (now in mypy config)

---
 sydent/http/matrixfederationagent.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sydent/http/matrixfederationagent.py b/sydent/http/matrixfederationagent.py
index 474c6654..a91b6090 100644
--- a/sydent/http/matrixfederationagent.py
+++ b/sydent/http/matrixfederationagent.py
@@ -18,7 +18,7 @@
 from typing import Optional, Tuple, Union
 
 import attr
-from netaddr import IPAddress  # type: ignore
+from netaddr import IPAddress
 from twisted.internet import defer
 from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
 from twisted.internet.interfaces import IStreamClientEndpoint

From e472538d97b56db23c645be5c2c4e06892659673 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 12:28:23 +0100
Subject: [PATCH 06/21] Annotate well_known_cache

---
 sydent/http/matrixfederationagent.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/sydent/http/matrixfederationagent.py b/sydent/http/matrixfederationagent.py
index a91b6090..efcc1ac8 100644
--- a/sydent/http/matrixfederationagent.py
+++ b/sydent/http/matrixfederationagent.py
@@ -49,7 +49,7 @@
 WELL_KNOWN_MAX_SIZE = 50 * 1024  # 50 KiB
 
 logger = logging.getLogger(__name__)
-well_known_cache = TTLCache("well-known")
+well_known_cache: TTLCache[bytes, Optional[bytes]] = TTLCache("well-known")
 
 
 @implementer(IAgent)
@@ -75,7 +75,6 @@ class MatrixFederationAgent:
 
     :param _well_known_cache: TTLCache impl for storing cached well-known
         lookups. Omit to use a default implementation.
-    :type _well_known_cache: TTLCache
     """
 
     def __init__(
@@ -84,7 +83,7 @@ def __init__(
         tls_client_options_factory,
         _well_known_tls_policy=None,
         _srv_resolver: Optional["SrvResolver"] = None,
-        _well_known_cache: "TTLCache" = well_known_cache,
+        _well_known_cache: TTLCache[bytes, Optional[bytes]] = well_known_cache,
     ) -> None:
         self._reactor = reactor
 

From c028ecc80f90bb8f346adc0511ac4f886f29948d Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 12:30:14 +0100
Subject: [PATCH 07/21] Annotate _parse_cache_control

---
 sydent/http/matrixfederationagent.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sydent/http/matrixfederationagent.py b/sydent/http/matrixfederationagent.py
index efcc1ac8..fbe95153 100644
--- a/sydent/http/matrixfederationagent.py
+++ b/sydent/http/matrixfederationagent.py
@@ -15,7 +15,7 @@
 import logging
 import random
 import time
-from typing import Optional, Tuple, Union
+from typing import Optional, Tuple, Union, Dict
 
 import attr
 from netaddr import IPAddress
@@ -402,8 +402,8 @@ def _cache_period_from_headers(headers, time_now=time.time):
     return None
 
 
-def _parse_cache_control(headers):
-    cache_controls = {}
+def _parse_cache_control(headers: Headers) -> Dict[bytes, Optional[bytes]]:
+    cache_controls: Dict[bytes, Optional[bytes]] = {}
     for hdr in headers.getRawHeaders(b"cache-control", []):
         for directive in hdr.split(b","):
             splits = [x.strip() for x in directive.split(b"=", 1)]

From 9126c47cf210da4c7668edd8e3bfb6002fcba98b Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 12:31:18 +0100
Subject: [PATCH 08/21] Annotate _cache_period_from_headers

---
 sydent/http/matrixfederationagent.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/sydent/http/matrixfederationagent.py b/sydent/http/matrixfederationagent.py
index fbe95153..0a244b03 100644
--- a/sydent/http/matrixfederationagent.py
+++ b/sydent/http/matrixfederationagent.py
@@ -15,7 +15,7 @@
 import logging
 import random
 import time
-from typing import Optional, Tuple, Union, Dict
+from typing import Optional, Tuple, Union, Dict, Callable
 
 import attr
 from netaddr import IPAddress
@@ -375,16 +375,18 @@ def connect(self, protocol_factory):
         return self.ep.connect(protocol_factory)
 
 
-def _cache_period_from_headers(headers, time_now=time.time):
+def _cache_period_from_headers(
+    headers: Headers, time_now: Callable[[], float] = time.time
+) -> Optional[float]:
     cache_controls = _parse_cache_control(headers)
 
     if b"no-store" in cache_controls:
         return 0
 
-    if b"max-age" in cache_controls:
+    max_age = cache_controls.get(b"max-age")
+    if max_age is not None:
         try:
-            max_age = int(cache_controls[b"max-age"])
-            return max_age
+            return int(max_age)
         except ValueError:
             pass
 

From d73280d338e57a4d713713f97edbdebfbb037526 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 12:31:27 +0100
Subject: [PATCH 09/21] Annotate LoggingHostnameEndpoint

---
 sydent/http/matrixfederationagent.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/sydent/http/matrixfederationagent.py b/sydent/http/matrixfederationagent.py
index 0a244b03..2b32ac79 100644
--- a/sydent/http/matrixfederationagent.py
+++ b/sydent/http/matrixfederationagent.py
@@ -15,13 +15,13 @@
 import logging
 import random
 import time
-from typing import Optional, Tuple, Union, Dict, Callable
+from typing import Optional, Tuple, Union, Dict, Callable, Any
 
 import attr
 from netaddr import IPAddress
 from twisted.internet import defer
 from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
-from twisted.internet.interfaces import IStreamClientEndpoint
+from twisted.internet.interfaces import IStreamClientEndpoint, IReactorTime, IProtocolFactory, IProtocol
 from twisted.web.client import URI, Agent, HTTPConnectionPool, RedirectAgent, Response
 from twisted.web.http import stringToDatetime
 from twisted.web.http_headers import Headers
@@ -364,13 +364,17 @@ async def _do_get_well_known(
 class LoggingHostnameEndpoint:
     """A wrapper for HostnameEndpint which logs when it connects"""
 
-    def __init__(self, reactor, host, port, *args, **kwargs):
+    def __init__(
+        self, reactor: IReactorTime, host: bytes, port: int, *args: Any, **kwargs: Any
+    ):
         self.host = host
         self.port = port
         self.ep = HostnameEndpoint(reactor, host, port, *args, **kwargs)
         logger.info("Endpoint created with %s:%d", host, port)
 
-    def connect(self, protocol_factory):
+    def connect(
+        self, protocol_factory: IProtocolFactory
+    ) -> "defer.Deferred[IProtocol]":
         logger.info("Connecting to %s:%i", self.host.decode("ascii"), self.port)
         return self.ep.connect(protocol_factory)
 

From 2b95cb1e40f86f293e87e2c17bce535d273fe9fb Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 12:34:04 +0100
Subject: [PATCH 10/21] Annotate _do_get_well_known

---
 sydent/http/matrixfederationagent.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/sydent/http/matrixfederationagent.py b/sydent/http/matrixfederationagent.py
index 2b32ac79..874735e5 100644
--- a/sydent/http/matrixfederationagent.py
+++ b/sydent/http/matrixfederationagent.py
@@ -309,7 +309,7 @@ async def _get_well_known(self, server_name: bytes) -> Optional[bytes]:
 
     async def _do_get_well_known(
         self, server_name: bytes
-    ) -> Tuple[Union[bytes, None, object], int]:
+    ) -> Tuple[Optional[bytes], float]:
         """Actually fetch and parse a .well-known, without checking the cache
 
         :param server_name: Name of the server, from the requested url
@@ -322,6 +322,7 @@ async def _do_get_well_known(
         uri = b"https://%s/.well-known/matrix/server" % (server_name,)
         uri_str = uri.decode("ascii")
         logger.info("Fetching %s", uri_str)
+        cache_period: Optional[float]
         try:
             response = await self._well_known_agent.request(b"GET", uri)
             body = await read_body_with_max_size(response, WELL_KNOWN_MAX_SIZE)
@@ -339,7 +340,7 @@ async def _do_get_well_known(
 
             # add some randomness to the TTL to avoid a stampeding herd every hour
             # after startup
-            cache_period: float = WELL_KNOWN_INVALID_CACHE_PERIOD
+            cache_period = WELL_KNOWN_INVALID_CACHE_PERIOD
             cache_period += random.uniform(0, WELL_KNOWN_DEFAULT_CACHE_PERIOD_JITTER)
             return (None, cache_period)
 

From c2bb3385c55b98b30dfae747f0eda07ad0c1416b Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 12:39:57 +0100
Subject: [PATCH 11/21] Workaround no annotation for Headers.copy

---
 sydent/http/matrixfederationagent.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/sydent/http/matrixfederationagent.py b/sydent/http/matrixfederationagent.py
index 874735e5..05cbc4b7 100644
--- a/sydent/http/matrixfederationagent.py
+++ b/sydent/http/matrixfederationagent.py
@@ -160,7 +160,10 @@ def request(
         if headers is None:
             headers = Headers()
         else:
-            headers = headers.copy()
+            # Type safety: Headers.copy doesn't have a return type annotated,
+            # and I don't want to stub web.http_headers. Could use stubgen? It's
+            # a pretty simple file.
+            headers = headers.copy()  # type: ignore[no-untyped-call]
             assert headers is not None
 
         if not headers.hasHeader(b"host"):

From cebe6df58431aaf01c0a738a883fd4125ce19f32 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 12:42:19 +0100
Subject: [PATCH 12/21] annotate EndpointFactory

---
 sydent/http/matrixfederationagent.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/sydent/http/matrixfederationagent.py b/sydent/http/matrixfederationagent.py
index 05cbc4b7..ffcec133 100644
--- a/sydent/http/matrixfederationagent.py
+++ b/sydent/http/matrixfederationagent.py
@@ -169,10 +169,11 @@ def request(
         if not headers.hasHeader(b"host"):
             headers.addRawHeader(b"host", routing.host_header)
 
+        @implementer(IAgentEndpointFactory)
         class EndpointFactory:
             @staticmethod
-            def endpointForURI(_uri):
-                ep = LoggingHostnameEndpoint(
+            def endpointForURI(_uri: URI) -> IStreamClientEndpoint:
+                ep: IStreamClientEndpoint = LoggingHostnameEndpoint(
                     self._reactor,
                     routing.target_host,
                     routing.target_port,

From c09ea607cb068e9fa017315aff2dc50d5675d953 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 12:43:28 +0100
Subject: [PATCH 13/21] Avoid str/bytes confusion in well_known handling

---
 sydent/http/matrixfederationagent.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/sydent/http/matrixfederationagent.py b/sydent/http/matrixfederationagent.py
index ffcec133..ba545e06 100644
--- a/sydent/http/matrixfederationagent.py
+++ b/sydent/http/matrixfederationagent.py
@@ -237,9 +237,11 @@ async def _route_matrix_uri(
                 # parse the server name in the .well-known response into host/port.
                 # (This code is lifted from twisted.web.client.URI.fromBytes).
                 if b":" in well_known_server:
-                    well_known_host, well_known_port = well_known_server.rsplit(b":", 1)
+                    well_known_host, well_known_port_raw = well_known_server.rsplit(
+                        b":", 1
+                    )
                     try:
-                        well_known_port = int(well_known_port)
+                        well_known_port = int(well_known_port_raw)
                     except ValueError:
                         # the part after the colon could not be parsed as an int
                         # - we assume it is an IPv6 literal with no port (the closing

From 32c00529c350792d62ada449b2d4aeac948b91f2 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 12:50:32 +0100
Subject: [PATCH 14/21] Annotations for MatrixFederationAgent

---
 sydent/http/matrixfederationagent.py | 40 +++++++++++++---------------
 1 file changed, 19 insertions(+), 21 deletions(-)

diff --git a/sydent/http/matrixfederationagent.py b/sydent/http/matrixfederationagent.py
index ba545e06..b4472be4 100644
--- a/sydent/http/matrixfederationagent.py
+++ b/sydent/http/matrixfederationagent.py
@@ -15,7 +15,7 @@
 import logging
 import random
 import time
-from typing import Optional, Tuple, Union, Dict, Callable, Any
+from typing import Optional, Tuple, Union, Dict, Callable, Any, Generator
 
 import attr
 from netaddr import IPAddress
@@ -25,9 +25,10 @@
 from twisted.web.client import URI, Agent, HTTPConnectionPool, RedirectAgent, Response
 from twisted.web.http import stringToDatetime
 from twisted.web.http_headers import Headers
-from twisted.web.iweb import IAgent, IBodyProducer
+from twisted.web.iweb import IAgent, IBodyProducer, IResponse, IAgentEndpointFactory, IPolicyForHTTPS
 from zope.interface import implementer
 
+from sydent.http.federation_tls_options import ClientTLSOptionsFactory
 from sydent.http.httpcommon import read_body_with_max_size
 from sydent.http.srvresolver import SrvResolver, pick_server_from_list
 from sydent.util import json_decoder
@@ -59,19 +60,12 @@ class MatrixFederationAgent:
     Doesn't implement any retries. (Those are done in MatrixFederationHttpClient.)
 
     :param reactor: twisted reactor to use for underlying requests
-    :type reactor: IReactor
 
     :param tls_client_options_factory: Factory to use for fetching client tls
         options, or none to disable TLS.
-    :type tls_client_options_factory: ClientTLSOptionsFactory, None
 
     :param _well_known_tls_policy: TLS policy to use for fetching .well-known
         files. None to use a default (browser-like) implementation.
-    :type _well_known_tls_policy: IPolicyForHTTPS, None
-
-    :param _srv_resolver: SRVResolver impl to use for looking up SRV records.
-        None to use a default implementation.
-    :type _srv_resolver: SrvResolver, None
 
     :param _well_known_cache: TTLCache impl for storing cached well-known
         lookups. Omit to use a default implementation.
@@ -79,10 +73,14 @@ class MatrixFederationAgent:
 
     def __init__(
         self,
-        reactor,
-        tls_client_options_factory,
-        _well_known_tls_policy=None,
-        _srv_resolver: Optional["SrvResolver"] = None,
+        # This reactor should also be IReactorTCP and IReactorPluggableNameResolver
+        # because it eventually makes its way to HostnameEndpoint.__init__.
+        # But that's not easy to express with an annotation. We use the
+        # `seconds` attribute below, so mark this as IReactorTime for now.
+        reactor: IReactorTime,
+        tls_client_options_factory: Optional[ClientTLSOptionsFactory],
+        _well_known_tls_policy: Optional[IPolicyForHTTPS] = None,
+        _srv_resolver: Optional[SrvResolver] = None,
         _well_known_cache: TTLCache[bytes, Optional[bytes]] = well_known_cache,
     ) -> None:
         self._reactor = reactor
@@ -97,15 +95,15 @@ def __init__(
         self._pool.maxPersistentPerHost = 5
         self._pool.cachedConnectionTimeout = 2 * 60
 
-        agent_args = {}
         if _well_known_tls_policy is not None:
             # the param is called 'contextFactory', but actually passing a
             # contextfactory is deprecated, and it expects an IPolicyForHTTPS.
-            agent_args["contextFactory"] = _well_known_tls_policy
-        _well_known_agent = RedirectAgent(
-            Agent(self._reactor, pool=self._pool, **agent_args),
-        )
-        self._well_known_agent = _well_known_agent
+            _well_known_agent = Agent(
+                self._reactor, pool=self._pool, contextFactory=_well_known_tls_policy
+            )
+        else:
+            _well_known_agent = Agent(self._reactor, pool=self._pool)
+        self._well_known_agent = RedirectAgent(_well_known_agent)
 
         # our cache of .well-known lookup results, mapping from server name
         # to delegated name. The values can be:
@@ -120,7 +118,7 @@ def request(
         uri: bytes,
         headers: Optional["Headers"] = None,
         bodyProducer: Optional["IBodyProducer"] = None,
-    ) -> Response:
+    ) -> Generator["defer.Deferred[Any]", Any, IResponse]:
         """
         :param method: HTTP method (GET/POST/etc).
 
@@ -183,7 +181,7 @@ def endpointForURI(_uri: URI) -> IStreamClientEndpoint:
                 return ep
 
         agent = Agent.usingEndpointFactory(self._reactor, EndpointFactory(), self._pool)
-        res = yield agent.request(method, uri, headers, bodyProducer)
+        res: IResponse = yield agent.request(method, uri, headers, bodyProducer)
         return res
 
     async def _route_matrix_uri(

From dc60506601085c247c8da980001ed790cf284117 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 13:01:44 +0100
Subject: [PATCH 15/21] Suppress reactor complaint for now

---
 sydent/http/httpclient.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/sydent/http/httpclient.py b/sydent/http/httpclient.py
index ca2182cb..7001f74f 100644
--- a/sydent/http/httpclient.py
+++ b/sydent/http/httpclient.py
@@ -177,7 +177,12 @@ class FederationHttpClient(HTTPClient[MatrixFederationAgent]):
     def __init__(self, sydent: "Sydent") -> None:
         self.sydent = sydent
         self.agent = MatrixFederationAgent(
-            BlacklistingReactorWrapper(
+            # Type-safety: I don't have a good way of expressing that
+            # the reactor is IReactorTCP, IReactorTime and
+            # IReactorPluggableNameResolver all at once. But it is, because
+            # it wraps the sydent reactor.
+            # TODO: can we introduce a SydentReactor type like SynapseReactor?
+            BlacklistingReactorWrapper(  # type: ignore[arg-type]
                 reactor=self.sydent.reactor,
                 ip_whitelist=sydent.config.general.ip_whitelist,
                 ip_blacklist=sydent.config.general.ip_blacklist,

From 33969ead6eac71f26154336bf5bd6b22d8934d97 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 13:07:12 +0100
Subject: [PATCH 16/21] sydent.http/*.py now passes mypy --strict

---
 pyproject.toml | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index f9b02b08..05722726 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -49,17 +49,11 @@ strict = true
 files = [
     # Find files that pass with
     #     find sydent tests -type d -not -name __pycache__ -exec bash -c "mypy --strict '{}' > /dev/null"  \; -print
+    # TODO "sydent/*.py"
     "sydent/config",
     "sydent/db",
-    "sydent/http/auth.py",
-    "sydent/http/blacklisting_reactor.py",
-    "sydent/http/federation_tls_options.py",
-    "sydent/http/httpclient.py",
-    "sydent/http/httpcommon.py",
-    "sydent/http/httpsclient.py",
-    "sydent/http/httpserver.py",
-    "sydent/http/matrixfederationagent.py",
-    "sydent/http/srvresolver.py",
+    "sydent/http/*.py",
+    # TODO "sydent/http/servlets",
     "sydent/hs_federation",
     "sydent/replication",
     "sydent/sms",

From c25ec8793e3a20ad90d8f43a71df7c192ec69a91 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 13:08:00 +0100
Subject: [PATCH 17/21] Isort

---
 sydent/http/matrixfederationagent.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/sydent/http/matrixfederationagent.py b/sydent/http/matrixfederationagent.py
index b4472be4..cb8b0731 100644
--- a/sydent/http/matrixfederationagent.py
+++ b/sydent/http/matrixfederationagent.py
@@ -21,11 +21,22 @@
 from netaddr import IPAddress
 from twisted.internet import defer
 from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
-from twisted.internet.interfaces import IStreamClientEndpoint, IReactorTime, IProtocolFactory, IProtocol
+from twisted.internet.interfaces import (
+    IStreamClientEndpoint,
+    IReactorTime,
+    IProtocolFactory,
+    IProtocol,
+)
 from twisted.web.client import URI, Agent, HTTPConnectionPool, RedirectAgent, Response
 from twisted.web.http import stringToDatetime
 from twisted.web.http_headers import Headers
-from twisted.web.iweb import IAgent, IBodyProducer, IResponse, IAgentEndpointFactory, IPolicyForHTTPS
+from twisted.web.iweb import (
+    IAgent,
+    IBodyProducer,
+    IResponse,
+    IAgentEndpointFactory,
+    IPolicyForHTTPS,
+)
 from zope.interface import implementer
 
 from sydent.http.federation_tls_options import ClientTLSOptionsFactory

From 48cd0471eed6c6d5ec34e2400c13dd6ad01d86c4 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 13:09:54 +0100
Subject: [PATCH 18/21] Changelog

---
 changelog.d/444.misc | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/444.misc

diff --git a/changelog.d/444.misc b/changelog.d/444.misc
new file mode 100644
index 00000000..4f884113
--- /dev/null
+++ b/changelog.d/444.misc
@@ -0,0 +1 @@
+Get `sydent.http.matrixfederationagent` to pass `mypy --strict`.
\ No newline at end of file

From c37e59e8f9b838873d24f2a3ed4bb322fc9f26a4 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 14:09:51 +0100
Subject: [PATCH 19/21] Additional linting --- looks like it didn't fully run?

---
 stubs/twisted/internet/endpoints.pyi |  6 +++---
 stubs/twisted/web/client.pyi         |  2 +-
 sydent/http/matrixfederationagent.py | 14 +++++++-------
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/stubs/twisted/internet/endpoints.pyi b/stubs/twisted/internet/endpoints.pyi
index f4be1022..2efb5017 100644
--- a/stubs/twisted/internet/endpoints.pyi
+++ b/stubs/twisted/internet/endpoints.pyi
@@ -1,11 +1,11 @@
-from typing import Optional, AnyStr, Any
+from typing import Any, AnyStr, Optional
 
 from twisted.internet import interfaces
 from twisted.internet.defer import Deferred
 from twisted.internet.interfaces import (
-    IProtocolFactory,
-    IProtocol,
     IOpenSSLClientConnectionCreator,
+    IProtocol,
+    IProtocolFactory,
     IStreamClientEndpoint,
 )
 from zope.interface import implementer
diff --git a/stubs/twisted/web/client.pyi b/stubs/twisted/web/client.pyi
index 2e1a5515..7b4be609 100644
--- a/stubs/twisted/web/client.pyi
+++ b/stubs/twisted/web/client.pyi
@@ -11,10 +11,10 @@ from twisted.internet.task import Cooperator
 from twisted.web.http_headers import Headers
 from twisted.web.iweb import (
     IAgent,
+    IAgentEndpointFactory,
     IBodyProducer,
     IPolicyForHTTPS,
     IResponse,
-    IAgentEndpointFactory,
 )
 from zope.interface import implementer
 
diff --git a/sydent/http/matrixfederationagent.py b/sydent/http/matrixfederationagent.py
index cb8b0731..676d669c 100644
--- a/sydent/http/matrixfederationagent.py
+++ b/sydent/http/matrixfederationagent.py
@@ -15,27 +15,27 @@
 import logging
 import random
 import time
-from typing import Optional, Tuple, Union, Dict, Callable, Any, Generator
+from typing import Any, Callable, Dict, Generator, Optional, Tuple
 
 import attr
 from netaddr import IPAddress
 from twisted.internet import defer
 from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS
 from twisted.internet.interfaces import (
-    IStreamClientEndpoint,
-    IReactorTime,
-    IProtocolFactory,
     IProtocol,
+    IProtocolFactory,
+    IReactorTime,
+    IStreamClientEndpoint,
 )
-from twisted.web.client import URI, Agent, HTTPConnectionPool, RedirectAgent, Response
+from twisted.web.client import URI, Agent, HTTPConnectionPool, RedirectAgent
 from twisted.web.http import stringToDatetime
 from twisted.web.http_headers import Headers
 from twisted.web.iweb import (
     IAgent,
-    IBodyProducer,
-    IResponse,
     IAgentEndpointFactory,
+    IBodyProducer,
     IPolicyForHTTPS,
+    IResponse,
 )
 from zope.interface import implementer
 

From a1632ef82e7f5e63ebd832ed6e467b3c3f064b9b Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 14:10:06 +0100
Subject: [PATCH 20/21] Keep 3.6 flake8 happy with annotations on previous line

---
 sydent/http/matrixfederationagent.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/sydent/http/matrixfederationagent.py b/sydent/http/matrixfederationagent.py
index 676d669c..d6dc2f55 100644
--- a/sydent/http/matrixfederationagent.py
+++ b/sydent/http/matrixfederationagent.py
@@ -149,9 +149,8 @@ def request(
             (including problems that prevent the request from being sent).
         """
         parsed_uri = URI.fromBytes(uri, defaultPort=-1)
-        routing: _RoutingResult = yield defer.ensureDeferred(
-            self._route_matrix_uri(parsed_uri)
-        )
+        routing: _RoutingResult
+        routing = yield defer.ensureDeferred(self._route_matrix_uri(parsed_uri))
 
         # set up the TLS connection params
         #
@@ -192,7 +191,8 @@ def endpointForURI(_uri: URI) -> IStreamClientEndpoint:
                 return ep
 
         agent = Agent.usingEndpointFactory(self._reactor, EndpointFactory(), self._pool)
-        res: IResponse = yield agent.request(method, uri, headers, bodyProducer)
+        res: IResponse
+        res = yield agent.request(method, uri, headers, bodyProducer)
         return res
 
     async def _route_matrix_uri(

From 0d9c2f714553079908f888990e2607f87b8fab50 Mon Sep 17 00:00:00 2001
From: David Robertson <davidr@element.io>
Date: Thu, 28 Oct 2021 15:29:11 +0100
Subject: [PATCH 21/21] Review fixup

---
 stubs/twisted/web/client.pyi         | 2 +-
 sydent/http/matrixfederationagent.py | 4 ----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/stubs/twisted/web/client.pyi b/stubs/twisted/web/client.pyi
index 7b4be609..fce41be2 100644
--- a/stubs/twisted/web/client.pyi
+++ b/stubs/twisted/web/client.pyi
@@ -29,7 +29,7 @@ class BrowserLikePolicyForHTTPS:
 class HTTPConnectionPool:
     persistent: bool
     maxPersistentPerHost: int
-    cachedConnectionTimeout: int  # (maybe float?)
+    cachedConnectionTimeout: float
     retryAutomatically: bool
     def __init__(self, reactor: object, persistent: bool = True): ...
 
diff --git a/sydent/http/matrixfederationagent.py b/sydent/http/matrixfederationagent.py
index d6dc2f55..ddd96a85 100644
--- a/sydent/http/matrixfederationagent.py
+++ b/sydent/http/matrixfederationagent.py
@@ -448,26 +448,22 @@ class _RoutingResult:
     """
     The value we should assign to the Host header (host:port from the matrix
     URI, or .well-known).
-    :type: bytes
     """
 
     tls_server_name: bytes
     """
     The server name we should set in the SNI (typically host, without port, from the
     matrix URI or .well-known)
-    :type: bytes
     """
 
     target_host: bytes
     """
     The hostname (or IP literal) we should route the TCP connection to (the target of the
     SRV record, or the hostname from the URL/.well-known)
-    :type: bytes
     """
 
     target_port: int
     """
     The port we should route the TCP connection to (the target of the SRV record, or
     the port from the URL/.well-known, or 8448)
-    :type: int
     """