Switch to Synapse >= 1.0

(Not mergable currently because of raiden-network#4634. Temporarily uses the base64
solution mentioned in the issue.)

This updates our testing dependency to matrix-synapse 1.2.1.

The Synapse config files generated by the integration tests have been
updated to align with new / changed options.

Additionally this no longer uses local TLS certificates speeding up
tests.

Fixes: raiden-network#3387
Required for: raiden-network#4292

raiden/network/transport/matrix/transport.py

@@ -486,7 +486,11 @@ def stop(self):
         self._client.api.session.close()
 
         self.log.debug("Matrix stopped", config=self._config)
-        del self.log
+        try:
+            del self.log
+        except AttributeError:
+            # During shutdown the log attribute may have already been collected
+            pass
         # parent may want to call get() after stop(), to ensure _run errors are re-raised
         # we don't call it here to avoid deadlock when self crashes and calls stop() on finally
 

raiden/network/transport/matrix/utils.py

@@ -37,6 +37,7 @@
 
 JOIN_RETRIES = 10
 USERID_RE = re.compile(r"^@(0x[0-9a-f]{40})(?:\.[0-9a-f]{8})?(?::.+)?$")
+DISPLAY_NAME_HEX_RE = re.compile(r"^0x[0-9a-fA-F]{130}$")
 ROOM_NAME_SEPARATOR = "_"
 ROOM_NAME_PREFIX = "raiden"
 
@@ -481,9 +482,10 @@ def login_or_register(
     else:
         raise ValueError("Could not register or login!")
 
-    name = encode_hex(signer.sign(client.user_id.encode()))
+    signature_bytes = signer.sign(client.user_id.encode())
+    signature_hex = encode_hex(signature_bytes)
     user = client.get_user(client.user_id)
-    user.set_display_name(name)
+    user.set_display_name(signature_hex)
     log.debug(
         "Matrix user login", homeserver=server_name, server_url=server_url, username=username
     )
@@ -503,9 +505,11 @@ def validate_userid_signature(user: User) -> Optional[Address]:
 
     try:
         displayname = user.get_display_name()
-        recovered = recover(
-            data=user.user_id.encode(), signature=Signature(decode_hex(displayname))
-        )
+        if DISPLAY_NAME_HEX_RE.match(displayname):
+            signature_bytes = decode_hex(displayname)
+        else:
+            return None
+        recovered = recover(data=user.user_id.encode(), signature=Signature(signature_bytes))
         if not (address and recovered and recovered == address):
             return None
     except (

raiden/tests/integration/network/transport/test_matrix_transport.py

@@ -913,6 +913,7 @@ def test_matrix_user_roaming(matrix_transports):
 
     transport0.start(raiden_service0, message_handler0, "")
     transport0.start_health_check(raiden_service1.address)
+
     with Timeout(TIMEOUT_MESSAGE_RECEIVE):
         while not is_reachable(transport1, raiden_service0.address):
             gevent.sleep(0.1)

raiden/tests/utils/synapse_config.yaml.template

@@ -1,24 +1,23 @@
-tls_certificate_path: "{server_dir}/localhost:{port}.tls.crt"
-tls_private_key_path: "{server_dir}/localhost:{port}.tls.key"
-tls_dh_params_path: "{server_dir}/localhost:{port}.tls.dh"
-
-no_tls: false
+no_tls: true
+federation_verify_certificates: false
+accept_keys_insecurely: true
 tls_fingerprints: []
+trusted_key_servers: []
+
 server_name: "localhost:{port}"
-web_client: true
 
 soft_file_limit: 0
 
 listeners:
   - port: {port}
-    tls: true
+    tls: false
     bind_addresses: ['127.0.0.1']
     type: http
 
     x_forwarded: false
 
     resources:
-      - names: [client, webclient]  # changed
+      - names: [client]  # changed
         compress: true
       - names: [federation]
         compress: false
@@ -28,16 +27,37 @@ database:
   args:
     database: ":memory:"  # changed
 
-event_cache_size: "10K"
+user_directory:
+  enabled: true
+  search_all_users: true
 
-rc_messages_per_second: 200000  # changed
-rc_message_burst_count: 100000  # changed
+event_cache_size: "10K"
 
-federation_rc_window_size: 1000
-federation_rc_sleep_limit: 10
-federation_rc_sleep_delay: 500
-federation_rc_reject_limit: 50
-federation_rc_concurrent: 3
+rc_message:
+  per_second:   50000  # changed
+  burst_count: 100000  # changed
+
+rc_registration:
+  per_second: 0.17
+  burst_count: 3
+
+rc_login:
+  address:
+    per_second: 100
+    burst_count: 100
+  account:
+    per_second: 100
+    burst_count: 100
+  failed_attempts:
+    per_second: 100
+    burst_count: 100
+
+rc_federation:
+  window_size: 1000
+  sleep_limit: 10
+  sleep_delay: 500
+  reject_limit: 50
+  concurrent: 3
 
 max_upload_size: "0M"  # changed
 max_image_pixels: "0M"  # changed
@@ -70,6 +90,9 @@ password_providers:
   - module: 'raiden.tests.utils.transport.EthAuthProvider'
     config:
       enabled: true
+  - module: 'raiden.tests.utils.transport.NoTLSFederationMonkeyPatchProvider'
+    config:
+      enabled: true
 
 # Whether to allow non server admins to create groups on this server
 enable_group_creation: false

raiden/tests/utils/transport.py

@@ -109,6 +109,39 @@ def parse_config(config):
         return config
 
 
+# Used from within synapse during tests
+class NoTLSFederationMonkeyPatchProvider:
+    """ Dummy auth provider that disables TLS on S2S federation.
+
+    This is used by the integration tests to avoid the need for tls certificates.
+    It's implemented as an auth provider since that's a handy way to inject code into the
+    synapse process.
+
+    It works by replacing ``synapse.crypto.context_factory.ClientTLSOptionsFactory`` with an
+    object that returns ``None`` when instantiated.
+    """
+
+    __version__ = "0.1"
+
+    class NoTLSFactory:
+        def __new__(cls, *args, **kwargs):  # pylint: disable=unused-argument
+            return None
+
+    def __init__(self, config, account_handler):  # pylint: disable=unused-argument
+        pass
+
+    @defer.inlineCallbacks
+    def check_password(self, user_id, password):  # pylint: disable=unused-argument,no-self-use
+        defer.returnValue(False)
+
+    @staticmethod
+    def parse_config(config):
+        from synapse.crypto import context_factory
+
+        context_factory.ClientTLSOptionsFactory = NoTLSFederationMonkeyPatchProvider.NoTLSFactory
+        return config
+
+
 def make_requests_insecure():
     """
     Prevent `requests` from performing TLS verification.
@@ -179,9 +212,17 @@ def matrix_server_starter(
         server_urls: List[ParsedURL] = []
         for _, port in zip(range(count), free_port_generator):
             server_name, config_file = config_generator(port)
-            server_url = ParsedURL(f"https://{server_name}")
+            server_url = ParsedURL(f"http://{server_name}")
             server_urls.append(server_url)
 
+            synapse_cmd = [
+                sys.executable,
+                "-m",
+                "synapse.app.homeserver",
+                f"--server-name={server_name}",
+                f"--config-path={config_file!s}",
+            ]
+
             synapse_io: EXECUTOR_IO = DEVNULL
             # Used in CI to capture the logs for failure analysis
             if _SYNAPSE_LOGS_PATH is not None:
@@ -195,6 +236,7 @@ def matrix_server_starter(
                     header = f"{header}: {log_context}"
                 header = f" {header} "
                 log_file.write(f"{header:=^100}\n")
+                log_file.write(f"Cmd: `{' '.join(synapse_cmd)}`\n")
                 log_file.flush()
 
                 synapse_io = DEVNULL, log_file, STDOUT
@@ -203,13 +245,7 @@ def matrix_server_starter(
             sleep = 0.1
 
             executor = HTTPExecutor(
-                [
-                    sys.executable,
-                    "-m",
-                    "synapse.app.homeserver",
-                    f"--server-name={server_name}",
-                    f"--config-path={config_file!s}",
-                ],
+                synapse_cmd,
                 url=urljoin(server_url, "/_matrix/client/versions"),
                 method="GET",
                 timeout=startup_timeout,

requirements/requirements-ci.txt

@@ -37,7 +37,6 @@ cryptography==2.7
 cytoolz==0.9.0.1
 daemonize==2.5.0
 decorator==4.4.0
-defusedxml==0.6.0
 docutils==0.14
 entrypoints==0.3
 eth-abi==1.3.0
@@ -77,22 +76,19 @@ itsdangerous==1.1.0
 jinja2==2.10.1
 jsonschema==3.0.1
 lazy-object-proxy==1.4.1
-ldap3==2.6
 lru-dict==1.1.6
 macholib==1.11            # via pyinstaller
 markupsafe==1.1.1
 marshmallow-dataclass==6.0.0rc4
 marshmallow-enum==1.5.1
 marshmallow-polyfield==5.7
 marshmallow==3.0.0rc8
-matrix-angular-sdk==0.6.8
 matrix-client==0.3.2
-matrix-synapse-ldap3==0.1.3
-matrix-synapse==0.33.9
+matrix-synapse==1.4.0rc1
 mccabe==0.6.1
 mirakuru==1.1.0
 more-itertools==7.0.0
-msgpack-python==0.5.6
+msgpack==0.6.1
 mypy-extensions==0.4.1
 mypy==0.720
 netaddr==0.7.19
@@ -111,7 +107,6 @@ pip-tools==3.8.0
 pluggy==0.12.0
 prometheus-client==0.3.1
 psutil==5.6.3
-psycopg2==2.7.7
 ptyprocess==0.6.0
 py-ecc==1.4.7
 py-geth==2.1.0
@@ -127,19 +122,18 @@ pygments==2.4.2
 pyhamcrest==1.9.0
 pyinstaller==3.4
 pylint==2.3.1
-pymacaroons-pynacl==0.9.3
+pymacaroons==0.13.0
 pynacl==1.3.0
 pyopenssl==19.0.0
 pyparsing==2.4.0
 pyrsistent==0.15.2
-pysaml2==4.7.0
 pysha3==1.0.2
 pytest-forked==1.0.2
 pytest-random==0.2
 pytest-select==0.1.2
 pytest-xdist==1.28.0
 pytest==4.6.3
-python-dateutil==2.8.0
+python-dateutil==2.8.0    # via s3cmd
 python-magic==0.4.15      # via s3cmd
 pytoml==0.1.20
 pytz==2019.1

requirements/requirements-dev.in

@@ -43,9 +43,4 @@ coverage
 bump2version
 
 # Test support
-matrix-synapse==0.33.9
-
-# Pin psycopg2 to prevent having to compile the c-extension
-# (see https://github.com/raiden-network/raiden/issues/3745)
-# can be removed once https://github.com/raiden-network/raiden/issues/3387
-psycopg2<2.8
+matrix-synapse==1.4.0rc1

requirements/requirements-dev.txt

@@ -32,11 +32,10 @@ colorama==0.4.1
 colour==0.1.5
 constantly==15.1.0        # via twisted
 coverage==4.5.3
-cryptography==2.7         # via pyopenssl, pysaml2, service-identity
+cryptography==2.7         # via pyopenssl, service-identity
 cytoolz==0.9.0.1
 daemonize==2.5.0          # via matrix-synapse
 decorator==4.4.0
-defusedxml==0.6.0         # via pysaml2
 docutils==0.14
 entrypoints==0.3          # via flake8
 eth-abi==1.3.0
@@ -75,21 +74,18 @@ itsdangerous==1.1.0
 jinja2==2.10.1
 jsonschema==3.0.1         # via matrix-synapse
 lazy-object-proxy==1.4.1  # via astroid
-ldap3==2.6                # via matrix-synapse-ldap3
 lru-dict==1.1.6
 markupsafe==1.1.1
 marshmallow-dataclass==6.0.0rc4
 marshmallow-enum==1.5.1
 marshmallow-polyfield==5.7
 marshmallow==3.0.0rc8
-matrix-angular-sdk==0.6.8  # via matrix-synapse
 matrix-client==0.3.2
-matrix-synapse-ldap3==0.1.3  # via matrix-synapse
-matrix-synapse==0.33.9
+matrix-synapse==1.4.0rc1
 mccabe==0.6.1             # via flake8, pylint
 mirakuru==1.1.0
 more-itertools==7.0.0     # via pytest
-msgpack-python==0.5.6     # via matrix-synapse
+msgpack==0.6.1            # via matrix-synapse
 mypy-extensions==0.4.1
 mypy==0.720
 netaddr==0.7.19           # via matrix-synapse
@@ -107,34 +103,31 @@ pip-tools==3.8.0
 pluggy==0.12.0            # via pytest
 prometheus-client==0.3.1  # via matrix-synapse
 psutil==5.6.3
-psycopg2==2.7.7
 ptyprocess==0.6.0
 py-ecc==1.4.7
 py-geth==2.1.0
 py-solc==3.2.0
 py==1.8.0                 # via pytest
 pyasn1-modules==0.2.5     # via matrix-synapse, service-identity
-pyasn1==0.4.5             # via ldap3, matrix-synapse, pyasn1-modules, service-identity
+pyasn1==0.4.5             # via matrix-synapse, pyasn1-modules, service-identity
 pycodestyle==2.5.0        # via autopep8, flake8
 pycparser==2.19
 pycryptodome==3.8.2
 pyflakes==2.1.1           # via flake8
 pygments==2.4.2
 pyhamcrest==1.9.0         # via twisted
 pylint==2.3.1
-pymacaroons-pynacl==0.9.3  # via matrix-synapse
-pynacl==1.3.0             # via matrix-synapse, pymacaroons-pynacl, signedjson
-pyopenssl==19.0.0         # via matrix-synapse, pysaml2, twisted
+pymacaroons==0.13.0       # via matrix-synapse
+pynacl==1.3.0             # via matrix-synapse, pymacaroons, signedjson
+pyopenssl==19.0.0         # via matrix-synapse, twisted
 pyparsing==2.4.0
 pyrsistent==0.15.2        # via jsonschema
-pysaml2==4.7.0            # via matrix-synapse
 pysha3==1.0.2
 pytest-forked==1.0.2      # via pytest-xdist
 pytest-random==0.2
 pytest-select==0.1.2
 pytest-xdist==1.28.0
 pytest==4.6.3
-python-dateutil==2.8.0    # via pysaml2
 pytoml==0.1.20
 pytz==2019.1
 pyyaml==5.1.1             # via matrix-synapse
@@ -146,7 +139,7 @@ requests==2.22.0
 rlp==1.1.0
 semantic-version==2.6.0
 semver==2.8.1
-service-identity==18.1.0  # via matrix-synapse, matrix-synapse-ldap3, twisted
+service-identity==18.1.0  # via matrix-synapse, twisted
 signedjson==1.0.0         # via matrix-synapse
 simplegeneric==0.8.1
 simplejson==3.16.0        # via canonicaljson
@@ -164,7 +157,7 @@ toml==0.10.0              # via black
 toolz==0.9.0
 traitlets==4.3.2
 treq==18.6.0              # via matrix-synapse
-twisted[tls]==19.2.1      # via matrix-synapse, matrix-synapse-ldap3, treq
+twisted[tls]==19.2.1      # via matrix-synapse, treq
 typed-ast==1.4.0          # via astroid, mypy
 typing-extensions==3.7.4
 typing-inspect==0.4.0