diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ae27b9529..d84baf8a66 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,7 +17,7 @@ templates: description: "resource type for underlying VM" default: "medium" type: enum - enum: ["small", "medium", "large", "xlarge"] + enum: ["small", "medium", "medium+", "large", "xlarge"] transport-layer: description: "transmission protocol used, udp or matrix" default: "udp" @@ -36,7 +36,7 @@ executors: description: "resource type for underlying VM" default: "medium" type: enum - enum: ["small", "medium", "large", "xlarge"] + enum: ["small", "medium", "medium+", "large", "xlarge"] working_directory: ~/raiden docker: - image: circleci/python:<< parameters.py-version >> @@ -121,28 +121,6 @@ jobs: - .local - raiden - build-synapse: - executor: - name: base-executor - py-version: "2.7" - steps: - - attach_workspace: - at: '~' - - restore_cache: - key: synapse-cache-{{ checksum "~/raiden/tools/install_synapse.sh" }} - - run: - name: installing synapse - command: | - tools/install_synapse.sh - - save_cache: - key: synapse-cache-{{ checksum "~/raiden/tools/install_synapse.sh" }} - paths: - - ~/raiden/.synapse - - persist_to_workspace: - root: '~' - paths: - - raiden/.synapse/ - prepare-python: parameters: py-version: @@ -283,6 +261,8 @@ jobs: BLOCKCHAIN_TYPE_ARG: "--blockchain-type=<< parameters.blockchain-type >>" TRANSPORT_OPTION: << parameters.transport-layer >> TRANSPORT_OPTION_ARG: "--transport=<< parameters.transport-layer >>" + RAIDEN_TESTS_SYNAPSE_BASE_DIR: /home/circleci/.cache/synapse + RAIDEN_TESTS_SYNAPSE_LOGS_DIR: /tmp/synapse-logs parallelism: << parameters.parallelism >> @@ -292,6 +272,10 @@ jobs: - config-path - restore_cache: key: ethash-{{ checksum "~/.local/bin/geth" }} + - restore_cache: + keys: + - synapse-keys-v1 + - synapse-keys- - run: name: Running tests #coverage run -m pytest \ @@ -313,11 +297,21 @@ jobs: paths: - "~/.ethash" + - save_cache: + key: synapse-keys-v1 + paths: + - "~/.cache/synapse" + - store_test_results: path: test-reports - store_artifacts: path: test-reports + destination: test-reports + + - store_artifacts: + path: /tmp/synapse-logs + destination: synapse-logs finalize: executor: @@ -415,10 +409,6 @@ workflows: jobs: - prepare-system - - build-synapse: - requires: - - prepare-system - - prepare-python: name: prepare-python-3.6 py-version: '3.6' @@ -442,11 +432,6 @@ workflows: py-version: "3.6" requires: - prepare-python-3.6 - filters: - tags: - only: /^v\d+\.\d+\.\d+$/ - branches: - only: /.*/ - smoketest: name: smoketest-udp-3.6 @@ -459,10 +444,8 @@ workflows: name: smoketest-matrix-3.6 py-version: "3.6" transport-layer: "matrix" - additional-args: "--local-matrix='~/raiden/.synapse/run_synapse.sh'" requires: - prepare-python-3.6 - - build-synapse - test: name: test-unit-3.6 @@ -471,7 +454,6 @@ workflows: blockchain-type: "geth" requires: - prepare-python-3.6 - - build-synapse - finalize: requires: @@ -489,7 +471,6 @@ workflows: - test: name: test-integration-udp-3.6 py-version: "3.6" - resource: "large" test-type: "integration" blockchain-type: "geth" transport-layer: "udp" @@ -500,11 +481,9 @@ workflows: - test: name: test-integration-matrix-3.6 py-version: "3.6" - resource: "large" test-type: "integration" blockchain-type: "geth" transport-layer: "matrix" - additional-args: "--local-matrix='~/raiden/.synapse/run_synapse.sh'" parallelism: 16 requires: - smoketest-matrix-3.6 @@ -520,11 +499,6 @@ workflows: py-version: "3.7" requires: - prepare-python-3.7 - filters: - tags: - only: /^v\d+\.\d+\.\d+$/ - branches: - only: /.*/ - smoketest: name: smoketest-udp-3.7 @@ -537,10 +511,8 @@ workflows: name: smoketest-matrix-3.7 py-version: "3.7" transport-layer: "matrix" - additional-args: "--local-matrix='~/raiden/.synapse/run_synapse.sh'" requires: - prepare-python-3.7 - - build-synapse - test: name: test-unit-3.7 @@ -549,11 +521,9 @@ workflows: blockchain-type: "geth" requires: - prepare-python-3.7 - - build-synapse - test: name: test-integration-udp-3.7 - resource: "large" py-version: "3.7" test-type: "integration" blockchain-type: "geth" @@ -565,11 +535,9 @@ workflows: - test: name: test-integration-matrix-3.7 py-version: "3.7" - resource: "large" test-type: "integration" blockchain-type: "geth" transport-layer: "matrix" - additional-args: "--local-matrix='~/raiden/.synapse/run_synapse.sh'" parallelism: 16 requires: - smoketest-matrix-3.7 @@ -621,10 +589,6 @@ workflows: jobs: - prepare-system - - build-synapse: - requires: - - prepare-system - - prepare-python: name: prepare-python-3.6 py-version: '3.6' @@ -660,10 +624,8 @@ workflows: name: smoketest-matrix-3.6 py-version: "3.6" transport-layer: "matrix" - additional-args: "--local-matrix='~/raiden/.synapse/run_synapse.sh'" requires: - prepare-python-3.6 - - build-synapse - test: name: test-unit-3.6 @@ -672,7 +634,6 @@ workflows: blockchain-type: "geth" requires: - prepare-python-3.6 - - build-synapse - finalize: requires: @@ -690,7 +651,6 @@ workflows: - test: name: test-integration-udp-3.6 py-version: "3.6" - resource: "large" test-type: "integration" blockchain-type: "geth" transport-layer: "udp" @@ -701,11 +661,9 @@ workflows: - test: name: test-integration-matrix-3.6 py-version: "3.6" - resource: "large" test-type: "integration" blockchain-type: "geth" transport-layer: "matrix" - additional-args: "--local-matrix='~/raiden/.synapse/run_synapse.sh'" parallelism: 16 requires: - smoketest-matrix-3.6 @@ -733,10 +691,8 @@ workflows: name: smoketest-matrix-3.7 py-version: "3.7" transport-layer: "matrix" - additional-args: "--local-matrix='~/raiden/.synapse/run_synapse.sh'" requires: - prepare-python-3.7 - - build-synapse - test: name: test-unit-3.7 @@ -745,12 +701,10 @@ workflows: blockchain-type: "geth" requires: - prepare-python-3.7 - - build-synapse - test: name: test-integration-udp-3.7 py-version: "3.7" - resource: "large" test-type: "integration" blockchain-type: "geth" transport-layer: "udp" @@ -761,11 +715,9 @@ workflows: - test: name: test-integration-matrix-3.7 py-version: "3.7" - resource: "large" test-type: "integration" blockchain-type: "geth" transport-layer: "matrix" - additional-args: "--local-matrix='~/raiden/.synapse/run_synapse.sh'" parallelism: 16 requires: - smoketest-matrix-3.7 diff --git a/.travis.yml b/.travis.yml index 51ac30fa6c..af6879dcd1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -102,7 +102,7 @@ jobs: if: (type = cron OR NOT commit_message =~ /\[ci / OR commit_message =~ /\[ci integration-general\]/) env: - TEST='raiden/tests/integration --ignore=raiden/tests/integration/transfer --ignore=raiden/tests/integration/long_running --ignore=raiden/tests/integration/api --ignore=raiden/tests/integration/contracts --ignore=raiden/tests/integration/cli' - - TRANSPORT_OPTIONS='--transport=matrix --local-matrix=${HOME}/.bin/run_synapse.sh' + - TRANSPORT_OPTIONS='--transport=matrix' - RUN_SYNAPSE=1 - stage: integration @@ -117,7 +117,7 @@ jobs: if: (type = cron OR NOT commit_message =~ /\[ci / OR commit_message =~ /\[ci integration-transfer\]/) env: - TEST='raiden/tests/integration/transfer' - - TRANSPORT_OPTIONS='--transport=matrix --local-matrix=${HOME}/.bin/run_synapse.sh' + - TRANSPORT_OPTIONS='--transport=matrix' - RUN_SYNAPSE=1 - stage: integration @@ -133,7 +133,7 @@ jobs: if: (type = cron OR NOT commit_message =~ /\[ci / OR commit_message =~ /\[ci integration-long-running\]/) env: - TEST='raiden/tests/integration/long_running' - - TRANSPORT_OPTIONS='--transport=matrix --local-matrix=${HOME}/.bin/run_synapse.sh' + - TRANSPORT_OPTIONS='--transport=matrix' - RUN_SYNAPSE=1 - RUN_COVERAGE=no_coverage @@ -155,7 +155,7 @@ jobs: if: (type = cron OR NOT commit_message =~ /\[ci / OR commit_message =~ /\[ci integration-api\]/) env: - TEST='raiden/tests/integration/api' - - TRANSPORT_OPTIONS='--transport=matrix --local-matrix=${HOME}/.bin/run_synapse.sh' + - TRANSPORT_OPTIONS='--transport=matrix' - RUN_SYNAPSE=1 - stage: integration @@ -163,7 +163,7 @@ jobs: if: (type = cron OR NOT commit_message =~ /\[ci / OR commit_message =~ /\[ci integration-cli\]/) env: - TEST='raiden/tests/integration/cli' - - TRANSPORT_OPTIONS='--transport=matrix --local-matrix=${HOME}/.bin/run_synapse.sh' + - TRANSPORT_OPTIONS='--transport=matrix' - RUN_SYNAPSE=1 - stage: deploy @@ -233,7 +233,7 @@ jobs: # pdbpp interferes with pyinstaller, uninstall it - pip uninstall -y pdbpp before_script: - - python setup.py build + - python setup.py build script: - source .travis/set_tag.sh - mkdir -p dist/archive diff --git a/.travis/before_install.sh b/.travis/before_install.sh index 45d071741d..bac1dd8110 100755 --- a/.travis/before_install.sh +++ b/.travis/before_install.sh @@ -7,7 +7,3 @@ set -x .travis/download_solc.sh .travis/download_geth.sh - -if [ "$RUN_SYNAPSE" ]; then - tools/install_synapse.sh -fi diff --git a/.travis/run_smoketest.sh b/.travis/run_smoketest.sh index b071cc4494..58465a9d2a 100755 --- a/.travis/run_smoketest.sh +++ b/.travis/run_smoketest.sh @@ -6,5 +6,5 @@ set -x if [[ -z ${RUN_SYNAPSE} ]]; then raiden --transport=udp smoketest else - raiden --transport=matrix smoketest --local-matrix="${HOME}/.bin/run_synapse.sh" + raiden --transport=matrix smoketest fi diff --git a/constraints.txt b/constraints.txt index 03ddec44fd..9adbc0b624 100644 --- a/constraints.txt +++ b/constraints.txt @@ -3,16 +3,22 @@ asn1crypto==0.24.0 atomicwrites==1.2.1 attrdict==2.0.0 attrs==18.2.0 +Automat==0.7.0 +bcrypt==3.1.5 cachetools==2.1.0 +canonicaljson==1.1.4 certifi==2018.8.24 cffi==1.11.5 chardet==3.0.4 click==6.7 coincurve==8.0.2 colorama==0.3.9 +constantly==15.1.0 cryptography==2.3.1 cytoolz==0.9.0.1 +daemonize==2.4.7 decorator==4.3.0 +defusedxml==0.5.0 eth-abi==1.2.0 eth-account==0.3.0 eth-bloom==1.0.1 @@ -24,45 +30,66 @@ eth-tester==0.1.0b32 eth-typing==1.3.0 eth-utils==1.2.1 filelock==3.0.8 -Flask==1.0.2 Flask-Cors==3.0.6 Flask-RESTful==0.3.6 +Flask==1.0.2 +frozendict==1.2 gevent==1.3.6 greenlet==0.4.15 hexbytes==0.1.0 +hyperlink==18.0.0 idna==2.7 -ipython==4.2.1 +incremental==17.5.0 ipython-genutils==0.2.0 +ipython==4.2.1 itsdangerous==0.24 Jinja2==2.10 jsonschema==2.6.0 +ldap3==2.5.1 +lmdb==0.94 lru-dict==1.1.6 MarkupSafe==1.0 -marshmallow==2.15.4 marshmallow-polyfield==3.2 +marshmallow==2.15.4 +matrix-angular-sdk==0.6.8 matrix-client==0.3.2 +matrix-synapse-ldap3==0.1.3 +matrix-synapse==0.33.9 miniupnpc==2.0.2 mirakuru==1.0.0 more-itertools==4.3.0 +msgpack-python==0.5.6 +netaddr==0.7.19 netifaces==0.10.7 networkx==2.1 parsimonious==0.8.0 pexpect==4.6.0 +phonenumbers==8.10.2 pickleshare==0.7.4 +pillow==5.3.0 pluggy==0.7.1 +prometheus-client==0.3.1 psutil==5.4.7 ptyprocess==0.6.0 -py==1.6.0 py-ecc==1.4.3 py-evm==0.2.0a32 py-geth==2.0.1 py-solc==3.1.0 +py==1.6.0 +pyasn1-modules==0.2.2 +pyasn1==0.4.4 pycparser==2.18 pycryptodome==3.6.6 pyethash==0.1.27 +PyHamcrest==1.9.0 +pymacaroons-pynacl==0.9.3 +pynacl==1.3.0 +pyopenssl==18.0.0 +pysaml2==4.6.5 pysha3==1.0.2 pystun-patched-for-raiden==0.1.0 pytest==3.10.1 +python-dateutil==2.7.5 pytoml==0.1.19 pytz==2018.5 raiden-contracts==0.8.0 @@ -71,14 +98,22 @@ raiden-webui==0.7.1 requests==2.20.0 rlp==1.0.2 semantic-version==2.6.0 +service-identity==18.1.0 +signedjson==1.0.0 simplegeneric==0.8.1 +simplejson==3.16.0 six==1.11.0 +sortedcontainers==2.1.0 structlog==18.2.0 toolz==0.9.0 traitlets==4.3.2 +treq==18.6.0 trie==1.3.8 +Twisted==18.9.0 +unpaddedbase64==1.1.0 urllib3==1.23 web3==4.7.1 webargs==4.0.0 websockets==6.0 Werkzeug==0.14.1 +zope.interface==4.6.0 diff --git a/raiden/network/transport/matrix.py b/raiden/network/transport/matrix.py index c6d814826e..8d20ba898e 100644 --- a/raiden/network/transport/matrix.py +++ b/raiden/network/transport/matrix.py @@ -291,7 +291,7 @@ def _http_retry_delay() -> Iterable[float]: while True: self._server_url: str = self._select_server(config) - self._server_name = config.get('server_name', urlparse(self._server_url).hostname) + self._server_name = config.get('server_name', urlparse(self._server_url).netloc) client_class = config.get('client_class', GMatrixClient) self._client: GMatrixClient = client_class( self._server_url, @@ -423,6 +423,10 @@ def stop(self): self._client.stop_listener_thread() # stop sync_thread, wait client's greenlets # wait own greenlets, no need to get on them, exceptions should be raised in _run() gevent.wait(self.greenlets + [r.greenlet for r in self._address_to_retrier.values()]) + + # Ensure keep-alive http connections are closed + self._client.api.session.close() + self.log.debug('Matrix stopped', config=self._config) del self.log # parent may want to call get() after stop(), to ensure _run errors are re-raised @@ -618,7 +622,7 @@ def _join_discovery_room(self): self._discovery_room_alias = self._make_room_alias(self._config['discovery_room']) servers = self._config.get('available_servers') or list() - servers = [urlparse(s).hostname for s in servers if urlparse(s).hostname] + servers = [urlparse(s).netloc for s in servers if urlparse(s).netloc] if self._server_name not in servers: servers.append(self._server_name) servers.sort(key=lambda s: '' if s == self._server_name else s) # our server goes first diff --git a/raiden/network/utils.py b/raiden/network/utils.py index 4a03fba52b..0361dfa9c8 100644 --- a/raiden/network/utils.py +++ b/raiden/network/utils.py @@ -1,47 +1,47 @@ -from itertools import count +import errno +import socket +from itertools import count, repeat +from socket import SocketKind from time import sleep from typing import Optional -import psutil import requests from requests import RequestException -def get_free_port(address: str, initial_port: int): - """Find an unused TCP port in a specified range. This should not - be used in misson-critical applications - a race condition may - occur if someone grabs the port before caller of this function - has chance to use it. - Parameters: - address : an ip address of interface to use - initial_port : port to start iteration with - Return: - Iterator that will return next unused port on a specified - interface +def get_free_port( + bind_address: str = '127.0.0.1', + initial_port: int = 0, + socket_kind: SocketKind = SocketKind.SOCK_STREAM, +): """ + Find an unused TCP port. + This should not be used in misson-critical applications - a race condition may occur if + someone grabs the port before caller of this function has chance to use it. - try: - # On OSX this function requires root privileges - psutil.net_connections() - except psutil.AccessDenied: - return count(initial_port) + If `initial_port` is passed the function will try to find a port as close as possible. + Otherwise a random port is chosen by the OS. - def _unused_ports(): - for port in count(initial_port): - # check if the port is being used - connect_using_port = ( - conn - for conn in psutil.net_connections() - if hasattr(conn, 'laddr') and - conn.laddr[0] == address and - conn.laddr[1] == port - ) + Returns an iterator that will return an unused port on the specified interface. + """ - # only generate unused ports - if not any(connect_using_port): - yield port + def _port_generator(): + if initial_port == 0: + next_port = repeat(0) + else: + next_port = count(start=initial_port) + for i in next_port: + sock = socket.socket(socket.AF_INET, socket_kind) + try: + sock.bind((bind_address, i)) + except OSError as ex: + if ex.errno == errno.EADDRINUSE: + continue + port = sock.getsockname()[1] + sock.close() + yield port - return _unused_ports() + return _port_generator() def get_http_rtt( diff --git a/raiden/tests/README.rst b/raiden/tests/README.rst index 708f240444..ce3250ce14 100644 --- a/raiden/tests/README.rst +++ b/raiden/tests/README.rst @@ -4,13 +4,3 @@ Running integration tests with different transport protocols Raiden can be run with two different underlying transport protocols, UDP and Matrix. Therefore tests that depend on the transport layer (all of them are integration tests) come in two versions. The pytest option ``--transport=none|udp|matrix|all`` can be used to specify which versions of the test to run. By default, only the UDP versions of the tests are run, since the Matrix versions require the local installation of the `Synapse `_ Matrix server. - -Installing Synapse for Matrix tests ------------------------------------ - -Synapse requires Python 2.7 and SQLite. Please run ``tools/install_synapse.sh`` to generate a standalone binary for it, or follow these steps: - -- create a Python 2 virtualenv, install Synapse in it with pip, and -- create a script that runs Synapse as a python module, using the configuration file in ``raiden/tests/test_files``. - -The script will then be called by the test suite during setup. The path can be specified with the ``--local-matrix`` option, it defaults to ``.synapse/run_synapse.sh`` in Raiden's main directory. diff --git a/raiden/tests/conftest.py b/raiden/tests/conftest.py index a7cc7a2244..02137640f6 100644 --- a/raiden/tests/conftest.py +++ b/raiden/tests/conftest.py @@ -1,17 +1,19 @@ # pylint: disable=wrong-import-position,redefined-outer-name,unused-wildcard-import,wildcard-import -import re -import sys - -import gevent -import py from gevent import monkey monkey.patch_all() if True: + import re + import sys + + import gevent + import py import pytest + from raiden.log_config import configure_logging from raiden.tests.fixtures.variables import * # noqa: F401,F403 + from raiden.tests.utils.transport import make_requests_insecure from raiden.utils.cli import LogLevelConfigType @@ -22,13 +24,6 @@ def pytest_addoption(parser): default='geth', ) - parser.addoption( - '--initial-port', - type=int, - default=29870, - help='Base port number used to avoid conflicts while running parallel tests.', - ) - parser.addoption( '--log-config', action='store', @@ -50,22 +45,6 @@ def pytest_addoption(parser): help='Run integration tests with udp, with matrix, with both or not at all.', ) - parser.addoption( - '--local-matrix', - dest='local_matrix', - default='.synapse/run_synapse.sh', - help="Command to run the local matrix server, or 'none', " - "default: '.synapse/run_synapse.sh'", - ) - - parser.addoption( - '--matrix-server', - action='store', - dest='matrix_server', - default='http://localhost:8008', - help="Host name of local matrix server if used, default: 'http://localhost:8008'", - ) - parser.addoption( '--gevent-monitoring-signal', action='store_true', @@ -188,6 +167,38 @@ def dont_exit_pytest(): gevent.get_hub().NOT_ERROR = (gevent.GreenletExit, SystemExit) +@pytest.fixture(scope='session', autouse=True) +def insecure_tls(): + make_requests_insecure() + + +# Convert `--transport all` to two separate invocations with `matrix` and `udp` +def pytest_generate_tests(metafunc): + if 'transport' in metafunc.fixturenames: + transport = metafunc.config.getoption('transport') + transport_and_privacy = list() + + # avoid collecting test if 'skip_if_not_*' + if transport in ('udp', 'all') and 'skip_if_not_matrix' not in metafunc.fixturenames: + transport_and_privacy.append(('udp', None)) + + if transport in ('matrix', 'all') and 'skip_if_not_udp' not in metafunc.fixturenames: + if 'public_and_private_rooms' in metafunc.fixturenames: + transport_and_privacy.extend([('matrix', False), ('matrix', True)]) + else: + transport_and_privacy.append(('matrix', False)) + + if 'private_rooms' in metafunc.fixturenames: + metafunc.parametrize('transport,private_rooms', transport_and_privacy) + else: + # If the test function isn't taking the `private_rooms` fixture only give the + # transport values + metafunc.parametrize( + 'transport', + list(set(transport_type for transport_type, _ in transport_and_privacy)), + ) + + if sys.platform == 'darwin': # On macOS the temp directory base path is already very long. # To avoid failures on ipc tests (ipc path length is limited to 104/108 chars on macOS/linux) diff --git a/raiden/tests/fixtures/variables.py b/raiden/tests/fixtures/variables.py index 1ffb5ef45a..018b4508f7 100644 --- a/raiden/tests/fixtures/variables.py +++ b/raiden/tests/fixtures/variables.py @@ -1,6 +1,7 @@ # pylint: disable=redefined-outer-name import os import random +from enum import Enum import pytest from eth_utils import denoms, remove_0x_prefix, to_normalized_address @@ -27,6 +28,11 @@ RED_EYES_PER_CHANNEL_PARTICIPANT_LIMIT = int(0.075 * 10 ** 18) +class TransportProtocol(Enum): + UDP = 'udp' + MATRIX = 'matrix' + + @pytest.fixture def settle_timeout(reveal_timeout): """ @@ -132,9 +138,9 @@ def channels_per_node(): @pytest.fixture -def retry_interval(request): - if request.config.option.transport == 'matrix': - return 5 +def retry_interval(transport_protocol): + if transport_protocol is TransportProtocol.MATRIX: + return 2 else: return 0.5 @@ -244,9 +250,9 @@ def blockchain_private_keys(blockchain_number_of_nodes, blockchain_key_seed): @pytest.fixture(scope='session') -def port_generator(request): +def port_generator(): """ count generator used to get a unique port number. """ - return get_free_port('127.0.0.1', request.config.option.initial_port) + return get_free_port('127.0.0.1') @pytest.fixture @@ -319,3 +325,30 @@ def environment_type(): def unrecoverable_error_should_crash(): """For testing an UnrecoverableError should crash""" return True + + +@pytest.fixture +def transport(request): + """ 'all' replaced by parametrize in conftest.pytest_generate_tests """ + return request.config.getoption('transport') + + +@pytest.fixture +def transport_protocol(transport): + return TransportProtocol(transport) + + +@pytest.fixture +def skip_if_not_udp(request): + """Skip the test if not run with UDP transport""" + if request.config.option.transport in ('udp', 'all'): + return + pytest.skip('This test works only with UDP transport') + + +@pytest.fixture +def skip_if_not_matrix(request): + """Skip the test if not run with Matrix transport""" + if request.config.option.transport in ('matrix', 'all'): + return + pytest.skip('This test works only with Matrix transport') diff --git a/raiden/tests/integration/api/test_pythonapi.py b/raiden/tests/integration/api/test_pythonapi.py index 7ef4042d81..ed4de22ed3 100644 --- a/raiden/tests/integration/api/test_pythonapi.py +++ b/raiden/tests/integration/api/test_pythonapi.py @@ -29,7 +29,7 @@ def test_token_addresses(raiden_network, token_addresses): @pytest.mark.parametrize('number_of_nodes', [2]) @pytest.mark.parametrize('channels_per_node', [0]) -def test_channel_lifecycle(raiden_network, token_addresses, deposit, transport_config): +def test_channel_lifecycle(raiden_network, token_addresses, deposit, transport_protocol): node1, node2 = raiden_network token_address = token_addresses[0] token_network_identifier = views.get_token_network_identifier_by_token_address( diff --git a/raiden/tests/integration/conftest.py b/raiden/tests/integration/conftest.py index 9d846fbf73..6a3457f4bf 100644 --- a/raiden/tests/integration/conftest.py +++ b/raiden/tests/integration/conftest.py @@ -3,21 +3,3 @@ from raiden.tests.integration.fixtures.smartcontracts import * # noqa: F401,F403 from raiden.tests.integration.fixtures.transport import * # noqa: F401,F403 from raiden_libs.test.fixtures.web3 import patch_genesis_gas_limit # noqa: F401, F403 - - -def pytest_generate_tests(metafunc): - if 'transport' in metafunc.fixturenames: - transport = metafunc.config.getoption('transport') - transport_and_privacy = list() - - # avoid collecting test if 'skip_if_not_*' - if transport in ('udp', 'all') and 'skip_if_not_matrix' not in metafunc.fixturenames: - transport_and_privacy.append(('udp', None)) - - if transport in ('matrix', 'all') and 'skip_if_not_udp' not in metafunc.fixturenames: - if 'public_and_private_rooms' in metafunc.fixturenames: - transport_and_privacy.extend([('matrix', False), ('matrix', True)]) - else: - transport_and_privacy.append(('matrix', False)) - - metafunc.parametrize('transport,private_rooms', transport_and_privacy) diff --git a/raiden/tests/integration/fixtures/raiden_network.py b/raiden/tests/integration/fixtures/raiden_network.py index 6678fd9bde..cf5cb4239a 100644 --- a/raiden/tests/integration/fixtures/raiden_network.py +++ b/raiden/tests/integration/fixtures/raiden_network.py @@ -37,7 +37,7 @@ def raiden_chain( nat_keepalive_timeout, environment_type, unrecoverable_error_should_crash, - local_matrix_server, + local_matrix_servers, private_rooms, retry_timeout, ): @@ -69,7 +69,7 @@ def raiden_chain( nat_keepalive_timeout=nat_keepalive_timeout, environment_type=environment_type, unrecoverable_error_should_crash=unrecoverable_error_should_crash, - local_matrix_url=local_matrix_server, + local_matrix_url=local_matrix_servers[0], private_rooms=private_rooms, ) @@ -142,7 +142,7 @@ def raiden_network( nat_keepalive_timeout, environment_type, unrecoverable_error_should_crash, - local_matrix_server, + local_matrix_servers, private_rooms, retry_timeout, ): @@ -166,7 +166,7 @@ def raiden_network( nat_keepalive_timeout=nat_keepalive_timeout, environment_type=environment_type, unrecoverable_error_should_crash=unrecoverable_error_should_crash, - local_matrix_url=local_matrix_server, + local_matrix_url=local_matrix_servers[0], private_rooms=private_rooms, ) diff --git a/raiden/tests/integration/fixtures/transport.py b/raiden/tests/integration/fixtures/transport.py index a8d0bcab6f..4b41dae9e9 100644 --- a/raiden/tests/integration/fixtures/transport.py +++ b/raiden/tests/integration/fixtures/transport.py @@ -1,86 +1,68 @@ -from collections import namedtuple -from enum import Enum -from urllib.parse import urljoin - import pytest -from raiden.utils.http import HTTPExecutor - -TransportConfig = namedtuple('TransportConfig', 'protocol parameters') -MatrixTransportConfig = namedtuple('MatrixTransportConfig', 'command server') - - -class TransportProtocol(Enum): - UDP = 'udp' - MATRIX = 'matrix' +from raiden.network.transport import MatrixTransport +from raiden.tests.fixtures.variables import TransportProtocol +from raiden.tests.utils.transport import generate_synapse_config, matrix_server_starter @pytest.fixture -def transport(request): - """ 'all' replaced by parametrize in conftest.pytest_generate_tests """ - return request.config.getoption('transport') +def public_and_private_rooms(): + """If present in a test, conftest.pytest_generate_tests will parametrize private_rooms fixture + """ + return True -@pytest.fixture -def transport_config(request, transport): - if transport == 'udp': - return TransportConfig(protocol=TransportProtocol.UDP, parameters=None) - elif transport == 'matrix': - command = request.config.getoption('local_matrix') - return TransportConfig( - protocol=TransportProtocol.MATRIX, - parameters=MatrixTransportConfig( - command=command, - server=request.config.getoption('matrix_server'), - ), - ) - else: - return None - # can be changed with command line options, see tests/conftest.py +@pytest.fixture(scope='session') +def synapse_config_generator(): + with generate_synapse_config() as generator: + yield generator @pytest.fixture -def skip_if_not_udp(request): - """Skip the test if not run with UDP transport""" - if request.config.option.transport in ('udp', 'all'): - return - pytest.skip('This test works only with UDP transport') +def matrix_server_count(): + return 1 @pytest.fixture -def skip_if_not_matrix(request): - """Skip the test if not run with Matrix transport""" - if request.config.option.transport in ('matrix', 'all'): +def local_matrix_servers( + request, + transport_protocol, + matrix_server_count, + synapse_config_generator, +): + if transport_protocol is not TransportProtocol.MATRIX: + yield [None] return - pytest.skip('This test works only with Matrix transport') - -@pytest.fixture -def public_and_private_rooms(): - """If present in a test, conftest.pytest_generate_tests will parametrize private_rooms fixture - """ - return True + starter = matrix_server_starter( + count=matrix_server_count, + config_generator=synapse_config_generator, + log_context=request.node.name, + ) + with starter as server_urls: + yield server_urls @pytest.fixture -def local_matrix_server(transport_config): - if not transport_config.protocol == TransportProtocol.MATRIX: - yield None - return +def matrix_transports(local_matrix_servers, retries_before_backoff, retry_interval, private_rooms): + transports = [] + for server in local_matrix_servers: + transports.append( + MatrixTransport({ + 'discovery_room': 'discovery', + 'retries_before_backoff': retries_before_backoff, + 'retry_interval': retry_interval, + 'server': server, + 'server_name': server.netloc, + 'available_servers': local_matrix_servers, + 'private_rooms': private_rooms, + }), + ) - server = transport_config.parameters.server + yield transports - # if command is none, assume server is already running - if transport_config.parameters.command in (None, 'none'): - yield server - return + for transport in transports: + transport.stop() - # otherwise, run our own local server - with HTTPExecutor( - transport_config.parameters.command, - url=urljoin(server, '/_matrix/client/versions'), - method='GET', - timeout=30, - shell=True, - ): - yield server + for transport in transports: + transport.get() diff --git a/raiden/tests/integration/test_matrix_transport.py b/raiden/tests/integration/test_matrix_transport.py index 306bcd0cb9..03cac589ae 100644 --- a/raiden/tests/integration/test_matrix_transport.py +++ b/raiden/tests/integration/test_matrix_transport.py @@ -4,6 +4,7 @@ import gevent import pytest +from gevent import Timeout from raiden.constants import UINT64_MAX from raiden.messages import Processed, SecretRequest @@ -21,12 +22,24 @@ USERID1 = '@Alice:Wonderland' +# All tests in this module require matrix +pytestmark = pytest.mark.usefixtures('skip_if_not_matrix') + + +class MessageHandler: + def __init__(self, bag: set): + self.bag = bag + + def on_message(self, _, message): + self.bag.add(message) + + @pytest.fixture def mock_matrix( monkeypatch, retry_interval, retries_before_backoff, - local_matrix_server, + local_matrix_servers, private_rooms, ): @@ -57,8 +70,8 @@ def mock_receive_delivered(klass, delivered): config = dict( retry_interval=retry_interval, retries_before_backoff=retries_before_backoff, - server=local_matrix_server, - server_name='matrix.local.raiden', + server=local_matrix_servers[0], + server_name=local_matrix_servers[0].netloc, available_servers=[], discovery_room='discovery', private_rooms=private_rooms, @@ -124,54 +137,46 @@ def make_message(convert_to_hex: bool = False, overwrite_data=None): return room, event -def test_normal_processing_hex(mock_matrix, skip_userid_validation, skip_if_not_matrix): +def test_normal_processing_hex(mock_matrix, skip_userid_validation): m = mock_matrix room, event = make_message(convert_to_hex=True) assert m._handle_message(room, event) -def test_normal_processing_json(mock_matrix, skip_userid_validation, skip_if_not_matrix): +def test_normal_processing_json(mock_matrix, skip_userid_validation): m = mock_matrix room, event = make_message(convert_to_hex=False) assert m._handle_message(room, event) -def test_processing_invalid_json(mock_matrix, skip_userid_validation, skip_if_not_matrix): +def test_processing_invalid_json(mock_matrix, skip_userid_validation): m = mock_matrix invalid_json = '{"foo": 1,' room, event = make_message(convert_to_hex=False, overwrite_data=invalid_json) assert not m._handle_message(room, event) -def test_sending_nonstring_body(mock_matrix, skip_userid_validation, skip_if_not_matrix): +def test_sending_nonstring_body(mock_matrix, skip_userid_validation): m = mock_matrix room, event = make_message(overwrite_data=b'somebinarydata') assert not m._handle_message(room, event) -def test_processing_invalid_message_json( - mock_matrix, - skip_userid_validation, - skip_if_not_matrix, -): +def test_processing_invalid_message_json(mock_matrix, skip_userid_validation): m = mock_matrix invalid_message = '{"this": 1, "message": 5, "is": 3, "not_valid": 5}' room, event = make_message(convert_to_hex=False, overwrite_data=invalid_message) assert not m._handle_message(room, event) -def test_processing_invalid_message_cmdid_json( - mock_matrix, - skip_userid_validation, - skip_if_not_matrix, -): +def test_processing_invalid_message_cmdid_json(mock_matrix, skip_userid_validation): m = mock_matrix invalid_message = '{"type": "NonExistentMessage", "is": 3, "not_valid": 5}' room, event = make_message(convert_to_hex=False, overwrite_data=invalid_message) assert not m._handle_message(room, event) -def test_processing_invalid_hex(mock_matrix, skip_userid_validation, skip_if_not_matrix): +def test_processing_invalid_hex(mock_matrix, skip_userid_validation): m = mock_matrix room, event = make_message(convert_to_hex=True) old_data = event['content']['body'] @@ -179,7 +184,7 @@ def test_processing_invalid_hex(mock_matrix, skip_userid_validation, skip_if_not assert not m._handle_message(room, event) -def test_processing_invalid_message_hex(mock_matrix, skip_userid_validation, skip_if_not_matrix): +def test_processing_invalid_message_hex(mock_matrix, skip_userid_validation): m = mock_matrix room, event = make_message(convert_to_hex=True) old_data = event['content']['body'] @@ -187,11 +192,7 @@ def test_processing_invalid_message_hex(mock_matrix, skip_userid_validation, ski assert not m._handle_message(room, event) -def test_processing_invalid_message_cmdid_hex( - mock_matrix, - skip_userid_validation, - skip_if_not_matrix, -): +def test_processing_invalid_message_cmdid_hex(mock_matrix, skip_userid_validation): m = mock_matrix room, event = make_message(convert_to_hex=True) old_data = event['content']['body'] @@ -200,8 +201,7 @@ def test_processing_invalid_message_cmdid_hex( def test_matrix_message_sync( - skip_if_not_matrix, - local_matrix_server, + local_matrix_servers, private_rooms, retry_interval, retries_before_backoff, @@ -210,8 +210,8 @@ def test_matrix_message_sync( 'discovery_room': 'discovery', 'retries_before_backoff': retries_before_backoff, 'retry_interval': retry_interval, - 'server': local_matrix_server, - 'server_name': 'matrix.local.raiden', + 'server': local_matrix_servers[0], + 'server_name': local_matrix_servers[0].netloc, 'available_servers': [], 'private_rooms': private_rooms, }) @@ -219,8 +219,8 @@ def test_matrix_message_sync( 'discovery_room': 'discovery', 'retries_before_backoff': retries_before_backoff, 'retry_interval': retry_interval, - 'server': local_matrix_server, - 'server_name': 'matrix.local.raiden', + 'server': local_matrix_servers[0], + 'server_name': local_matrix_servers[0].netloc, 'available_servers': [], 'private_rooms': private_rooms, }) @@ -229,12 +229,7 @@ def test_matrix_message_sync( received_messages = set() - class MessageHandler: - def on_message(self, _, message): - nonlocal received_messages - received_messages.add(message) - - message_handler = MessageHandler() + message_handler = MessageHandler(received_messages) raiden_service0 = MockRaidenService(message_handler) raiden_service1 = MockRaidenService(message_handler) @@ -312,8 +307,7 @@ def on_message(self, _, message): def test_matrix_message_retry( - skip_if_not_matrix, - local_matrix_server, + local_matrix_servers, private_rooms, retry_interval, retries_before_backoff, @@ -332,8 +326,8 @@ def test_matrix_message_retry( 'discovery_room': 'discovery', 'retries_before_backoff': retries_before_backoff, 'retry_interval': retry_interval, - 'server': local_matrix_server, - 'server_name': 'matrix.local.raiden', + 'server': local_matrix_servers[0], + 'server_name': local_matrix_servers[0].netloc, 'available_servers': [], 'private_rooms': private_rooms, }) @@ -396,8 +390,7 @@ def test_matrix_message_retry( def test_join_invalid_discovery( - skip_if_not_matrix, - local_matrix_server, + local_matrix_servers, private_rooms, retry_interval, retries_before_backoff, @@ -412,8 +405,8 @@ def test_join_invalid_discovery( 'discovery_room': 'discovery', 'retries_before_backoff': retries_before_backoff, 'retry_interval': retry_interval, - 'server': local_matrix_server, - 'server_name': 'matrix.local.raiden', + 'server': local_matrix_servers[0], + 'server_name': local_matrix_servers[0].netloc, 'available_servers': ['http://invalid.server'], 'private_rooms': private_rooms, }) @@ -433,3 +426,65 @@ def test_join_invalid_discovery( transport.stop() transport.get() + + +@pytest.mark.parametrize('matrix_server_count', [2]) +def test_matrix_cross_server(matrix_transports, retry_interval): + transport0, transport1 = matrix_transports + + received_messages0 = set() + received_messages1 = set() + + message_handler0 = MessageHandler(received_messages0) + message_handler1 = MessageHandler(received_messages1) + raiden_service0 = MockRaidenService(message_handler0) + raiden_service1 = MockRaidenService(message_handler1) + + transport0.start(raiden_service0, message_handler0, '') + transport1.start(raiden_service1, message_handler1, '') + + transport1.start_health_check(raiden_service0.address) + transport0.start_health_check(raiden_service1.address) + + queueid = QueueIdentifier( + recipient=raiden_service1.address, + channel_identifier=CHANNEL_IDENTIFIER_GLOBAL_QUEUE, + ) + message = Processed(0) + message.sign(raiden_service0.private_key) + + transport0.send_async(queueid, message) + + with Timeout(retry_interval * 10, exception=False): + while not (len(received_messages0) == 1 and len(received_messages1) == 1): + gevent.sleep(.1) + + assert len(received_messages0) == 1 + assert len(received_messages1) == 1 + + transport0.stop() + transport1.stop() + transport0.get() + transport1.get() + + +def test_matrix_discovery_room_offline_server( + local_matrix_servers, + retries_before_backoff, + retry_interval, + private_rooms, +): + + transport = MatrixTransport({ + 'discovery_room': 'discovery', + 'retries_before_backoff': retries_before_backoff, + 'retry_interval': retry_interval, + 'server': local_matrix_servers[0], + 'server_name': local_matrix_servers[0].netloc, + 'available_servers': [local_matrix_servers[0], 'https://localhost:1'], + 'private_rooms': private_rooms, + }) + transport.start(MockRaidenService(None), MessageHandler(set()), '') + gevent.sleep(.2) + transport.stop() + transport.get() diff --git a/raiden/tests/integration/test_regression.py b/raiden/tests/integration/test_regression.py index 9f983c1e40..239281533e 100644 --- a/raiden/tests/integration/test_regression.py +++ b/raiden/tests/integration/test_regression.py @@ -5,8 +5,8 @@ from raiden.constants import UINT64_MAX from raiden.messages import Lock, LockedTransfer, RevealSecret, Secret +from raiden.tests.fixtures.variables import TransportProtocol from raiden.tests.integration.fixtures.raiden_network import CHAIN, wait_for_channels -from raiden.tests.integration.fixtures.transport import TransportProtocol from raiden.tests.utils.events import must_contain_entry from raiden.tests.utils.factories import UNIT_CHAIN_ID from raiden.tests.utils.network import payment_channel_open_and_deposit @@ -86,7 +86,7 @@ def test_regression_unfiltered_routes( @pytest.mark.parametrize('number_of_nodes', [3]) @pytest.mark.parametrize('channels_per_node', [CHAIN]) -def test_regression_revealsecret_after_secret(raiden_network, token_addresses, transport_config): +def test_regression_revealsecret_after_secret(raiden_network, token_addresses, transport_protocol): """ A RevealSecret message received after a Secret message must be cleanly handled. """ @@ -122,11 +122,11 @@ def test_regression_revealsecret_after_secret(raiden_network, token_addresses, t ) app2.raiden.sign(reveal_secret) - if transport_config.protocol is TransportProtocol.UDP: + if transport_protocol is TransportProtocol.UDP: reveal_data = reveal_secret.encode() host_port = None app1.raiden.transport.receive(reveal_data, host_port) - elif transport_config.protocol is TransportProtocol.MATRIX: + elif transport_protocol is TransportProtocol.MATRIX: app1.raiden.transport._receive_message(reveal_secret) # pylint: disable=protected-access else: raise TypeError('Unknown TransportProtocol') @@ -134,7 +134,7 @@ def test_regression_revealsecret_after_secret(raiden_network, token_addresses, t @pytest.mark.parametrize('number_of_nodes', [2]) @pytest.mark.parametrize('channels_per_node', [CHAIN]) -def test_regression_multiple_revealsecret(raiden_network, token_addresses, transport_config): +def test_regression_multiple_revealsecret(raiden_network, token_addresses, transport_protocol): """ Multiple RevealSecret messages arriving at the same time must be handled properly. @@ -192,11 +192,11 @@ def test_regression_multiple_revealsecret(raiden_network, token_addresses, trans ) app0.raiden.sign(mediated_transfer) - if transport_config.protocol is TransportProtocol.UDP: + if transport_protocol is TransportProtocol.UDP: message_data = mediated_transfer.encode() host_port = None app1.raiden.transport.receive(message_data, host_port) - elif transport_config.protocol is TransportProtocol.MATRIX: + elif transport_protocol is TransportProtocol.MATRIX: app1.raiden.transport._receive_message(mediated_transfer) else: raise TypeError('Unknown TransportProtocol') @@ -222,7 +222,7 @@ def test_regression_multiple_revealsecret(raiden_network, token_addresses, trans ) app0.raiden.sign(secret) - if transport_config.protocol is TransportProtocol.UDP: + if transport_protocol is TransportProtocol.UDP: messages = [ secret.encode(), reveal_secret.encode(), @@ -238,7 +238,7 @@ def test_regression_multiple_revealsecret(raiden_network, token_addresses, trans ) for data in messages ] - elif transport_config.protocol is TransportProtocol.MATRIX: + elif transport_protocol is TransportProtocol.MATRIX: messages = [ secret, reveal_secret, diff --git a/raiden/tests/unit/test_udp_transport.py b/raiden/tests/unit/test_udp_transport.py index 7be78a744d..d14034e501 100644 --- a/raiden/tests/unit/test_udp_transport.py +++ b/raiden/tests/unit/test_udp_transport.py @@ -11,6 +11,8 @@ from raiden.tests.utils.mocks import MockRaidenService from raiden.tests.utils.transport import MockDiscovery +pytestmark = pytest.mark.usefixtures('skip_if_not_udp') + @pytest.fixture def mock_udp( diff --git a/raiden/tests/utils/geth.py b/raiden/tests/utils/geth.py index cabc26d24b..ba316273d0 100644 --- a/raiden/tests/utils/geth.py +++ b/raiden/tests/utils/geth.py @@ -127,6 +127,8 @@ def geth_create_account(datadir: str, privkey: bytes): ['geth', '--datadir', datadir, 'account', 'import', keyfile_path], stdin=subprocess.PIPE, universal_newlines=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, ) create.stdin.write(DEFAULT_PASSPHRASE + os.linesep) @@ -330,7 +332,6 @@ def geth_run_nodes( log_path = os.path.join(logdir, str(pos)) logfile = open(log_path, 'w') stdout = logfile - stderr = logfile if '--unlock' in cmd: process = subprocess.Popen( @@ -338,7 +339,7 @@ def geth_run_nodes( universal_newlines=True, stdin=subprocess.PIPE, stdout=stdout, - stderr=stderr, + stderr=subprocess.STDOUT, ) # --password wont work, write password to unlock @@ -349,7 +350,7 @@ def geth_run_nodes( cmd, universal_newlines=True, stdout=stdout, - stderr=stderr, + stderr=subprocess.STDOUT, ) processes_list.append(process) diff --git a/raiden/tests/utils/network.py b/raiden/tests/utils/network.py index 1cf9194581..c671b868c0 100644 --- a/raiden/tests/utils/network.py +++ b/raiden/tests/utils/network.py @@ -342,7 +342,7 @@ def create_apps( 'retries_before_backoff': retries_before_backoff, 'retry_interval': retry_interval, 'server': local_matrix_url, - 'server_name': 'matrix.local.raiden', + 'server_name': local_matrix_url.netloc, 'available_servers': [], 'private_rooms': private_rooms, }, diff --git a/raiden/tests/utils/smoketest.py b/raiden/tests/utils/smoketest.py index d5f38a959f..f4da0cdc94 100644 --- a/raiden/tests/utils/smoketest.py +++ b/raiden/tests/utils/smoketest.py @@ -196,7 +196,7 @@ def setup_testchain_and_raiden(transport, matrix_server, print_step, contracts_v ensure_executable('geth') - free_port = get_free_port('127.0.0.1', 27854) + free_port = get_free_port('127.0.0.1') rpc_port = next(free_port) p2p_port = next(free_port) base_datadir = os.environ['RST_DATADIR'] @@ -287,9 +287,6 @@ def setup_testchain_and_raiden(transport, matrix_server, print_step, contracts_v print_step('Setting up Raiden') - if matrix_server == 'auto': - matrix_server = 'http://localhost:8008' - endpoint_registry_contract_address = to_checksum_address( contract_addresses[CONTRACT_ENDPOINT_REGISTRY], ) diff --git a/tools/synapse-config.yaml b/raiden/tests/utils/synapse_config.yaml.template similarity index 77% rename from tools/synapse-config.yaml rename to raiden/tests/utils/synapse_config.yaml.template index 722170bb82..a5b85c5d14 100644 --- a/tools/synapse-config.yaml +++ b/raiden/tests/utils/synapse_config.yaml.template @@ -1,13 +1,17 @@ -no_tls: true +tls_certificate_path: "{server_dir}/localhost:{port}.tls.crt" +tls_private_key_path: "{server_dir}/localhost:{port}.tls.key" +tls_dh_params_path: "{server_dir}/localhost:{port}.tls.dh" + +no_tls: false tls_fingerprints: [] -server_name: "matrix.local.raiden" +server_name: "localhost:{port}" web_client: true soft_file_limit: 0 listeners: - - port: 8008 - tls: false + - port: {port} + tls: true bind_addresses: ['127.0.0.1'] type: http @@ -39,7 +43,7 @@ max_upload_size: "0M" # changed max_image_pixels: "0M" # changed url_preview_enabled: false -enable_registration: false # changed +enable_registration: true # changed bcrypt_rounds: 12 allow_guest_access: false enable_metrics: false @@ -56,14 +60,14 @@ room_invite_state_types: app_service_config_files: [] expire_access_token: false -old_signing_keys: {} +old_signing_keys: {{}} key_refresh_interval: "1d" # 1 Day. password_config: enabled: true password_providers: - - module: 'eth_auth_provider.EthAuthProvider' + - module: 'raiden.tests.utils.transport.EthAuthProvider' config: enabled: true diff --git a/raiden/tests/utils/transport.py b/raiden/tests/utils/transport.py index 3379713ed7..33cc3e7f5c 100644 --- a/raiden/tests/utils/transport.py +++ b/raiden/tests/utils/transport.py @@ -1,4 +1,226 @@ -class MockDiscovery(object): +import logging +import os +import re +import shutil +import subprocess +import sys +from binascii import unhexlify +from contextlib import ExitStack, contextmanager +from datetime import datetime +from pathlib import Path +from tempfile import mkdtemp +from typing import ContextManager +from urllib.parse import urljoin, urlsplit + +import requests +from twisted.internet import defer + +from raiden.network.utils import get_free_port +from raiden.utils.http import HTTPExecutor +from raiden.utils.signing import eth_recover +_SYNAPSE_BASE_DIR_VAR_NAME = 'RAIDEN_TESTS_SYNAPSE_BASE_DIR' +_SYNAPSE_LOGS_PATH = os.environ.get('RAIDEN_TESTS_SYNAPSE_LOGS_DIR', False) +_SYNAPSE_CONFIG_TEMPLATE = Path(__file__).parent.joinpath('synapse_config.yaml.template') + + +class MockDiscovery(object): def get(self, node_address: bytes): return '127.0.0.1:5252' + + +class ParsedURL(str): + """ A string subclass that allows direct access to the split components of a URL """ + def __new__(cls, *args, **kwargs): + new = str.__new__(cls, *args, **kwargs) + new._parsed = urlsplit(new) + return new + + def __dir__(self): + return dir('') + dir(self._parsed) + + def __repr__(self): + return f"<{self.__class__.__name__}('{self}')>" + + def __getattr__(self, item): + try: + return getattr(self._parsed, item) + except AttributeError: + raise AttributeError( + f"'{self.__class__.__name__}' object has no attribute '{item}'", + ) + + +# Used from within synapse during tests +class EthAuthProvider(object): + __version__ = '0.1' + _user_re = re.compile(r'^@(0x[0-9a-f]{40}):(.+)$') + _password_re = re.compile(r'^0x[0-9a-f]{130}$') + + def __init__(self, config, account_handler): + self.account_handler = account_handler + self.config = config + self.hs_hostname = self.account_handler.hs.hostname + self.log = logging.getLogger(__name__) + + @defer.inlineCallbacks + def check_password(self, user_id, password): + if not password: + self.log.error('no password provided, user=%r', user_id) + defer.returnValue(False) + + if not self._password_re.match(password): + self.log.error( + 'invalid password format, must be 0x-prefixed hex, ' + 'lowercase, 65-bytes hash. user=%r', + user_id, + ) + defer.returnValue(False) + + signature = unhexlify(password[2:]) + + user_match = self._user_re.match(user_id) + if not user_match or user_match.group(2) != self.hs_hostname: + self.log.error( + 'invalid user format, must start with 0x-prefixed hex, ' + 'lowercase address. user=%r', + user_id, + ) + defer.returnValue(False) + + user_addr_hex = user_match.group(1) + user_addr = unhexlify(user_addr_hex[2:]) + + rec_addr = eth_recover(data=self.hs_hostname.encode(), signature=signature) + if not rec_addr or rec_addr != user_addr: + self.log.error( + 'invalid account password/signature. user=%r, signer=%r', + user_id, + rec_addr, + ) + defer.returnValue(False) + + localpart = user_id.split(":", 1)[0][1:] + self.log.info('eth login! valid signature. user=%r', user_id) + + if not (yield self.account_handler.check_user_exists(user_id)): + self.log.info('first user login, registering: user=%r', user_id) + yield self.account_handler.register(localpart=localpart) + + defer.returnValue(True) + + @staticmethod + def parse_config(config): + return config + + +def make_requests_insecure(): + """ + Prevent `requests` from performing TLS verification. + + **THIS MUST ONLY BE USED FOR TESTING PURPOSES!** + """ + # Disable verification in requests by replacing the 'verify' + # attribute with non-writable property that always returns `False` + requests.Session.verify = property(lambda self: False, lambda self, val: None) + + +@contextmanager +def generate_synapse_config() -> ContextManager: + # Allows caching of self signed synapse certificates on CI systems + if _SYNAPSE_BASE_DIR_VAR_NAME in os.environ: + synapse_base_dir = Path(os.environ[_SYNAPSE_BASE_DIR_VAR_NAME]) + synapse_base_dir.mkdir(parents=True, exist_ok=True) + delete_base_dir = False + else: + synapse_base_dir = Path(mkdtemp(prefix='pytest-synapse-')) + delete_base_dir = True + + def generate_config(port: int): + server_dir = synapse_base_dir.joinpath(f'localhost-{port}') + server_dir.mkdir(parents=True, exist_ok=True) + + server_name = f'localhost:{port}' + + # Always overwrite config file to ensure we're not using a stale version + config_file = server_dir.joinpath('synapse_config.yaml').resolve() + config_template = _SYNAPSE_CONFIG_TEMPLATE.read_text() + config_file.write_text(config_template.format(server_dir=server_dir, port=port)) + + tls_key_file = server_dir.joinpath(f'{server_name}.tls.crt') + + if not tls_key_file.exists(): + subprocess.run( + [ + sys.executable, + '-m', + 'synapse.app.homeserver', + f'--server-name={server_name}', + f'--config-path={config_file!s}', + '--generate-keys', + ], + cwd=server_dir, + timeout=30, + check=True, + stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + ) + return server_name, config_file + + try: + yield generate_config + finally: + if delete_base_dir: + shutil.rmtree(synapse_base_dir) + + +@contextmanager +def matrix_server_starter( + *, + count: int = 1, + config_generator: ContextManager = None, + log_context: str = None, +) -> ContextManager: + with ExitStack() as exit_stack: + if config_generator is None: + config_generator = exit_stack.enter_context(generate_synapse_config()) + server_urls = [] + for _, port in zip(range(count), get_free_port(initial_port=8500)): + server_name, config_file = config_generator(port) + server_url = ParsedURL(f'https://{server_name}') + server_urls.append(server_url) + + synapse_io = subprocess.DEVNULL + # Used in CI to capture the logs for failure analysis + if _SYNAPSE_LOGS_PATH: + log_file_path = Path(_SYNAPSE_LOGS_PATH).joinpath(f'{server_name}.log') + log_file_path.parent.mkdir(parents=True, exist_ok=True) + log_file = exit_stack.enter_context(log_file_path.open('at')) + + # Preface log with header + header = datetime.utcnow().isoformat() + if log_context: + header = f'{header}: {log_context}' + header = f' {header} ' + log_file.write(f'{header:=^100}\n') + log_file.flush() + + synapse_io = subprocess.DEVNULL, log_file, subprocess.STDOUT + + exit_stack.enter_context( + HTTPExecutor( + [ + sys.executable, + '-m', + 'synapse.app.homeserver', + f'--server-name={server_name}', + f'--config-path={config_file.name}', + ], + url=urljoin(server_url, '/_matrix/client/versions'), + method='GET', + timeout=30, + cwd=config_file.parent, + io=synapse_io, + ), + ) + yield server_urls diff --git a/raiden/ui/cli.py b/raiden/ui/cli.py index 391bade6eb..33d8b66b82 100644 --- a/raiden/ui/cli.py +++ b/raiden/ui/cli.py @@ -1,18 +1,19 @@ +import contextlib import json import os import sys import textwrap import traceback from copy import deepcopy -from pathlib import Path -from subprocess import DEVNULL +from io import StringIO from tempfile import mktemp -from urllib.parse import urljoin import click import structlog +import urllib3 from eth_utils import to_canonical_address, to_checksum_address from mirakuru import ProcessExitedWithError +from urllib3.exceptions import InsecureRequestWarning from raiden.api.rest import APIServer, RestAPI from raiden.app import App @@ -22,6 +23,7 @@ from raiden.network.sockfactory import SocketFactory from raiden.network.utils import get_free_port from raiden.settings import INITIAL_PORT +from raiden.tests.utils.transport import make_requests_insecure, matrix_server_starter from raiden.utils import get_system_spec, merge_dict, split_endpoint from raiden.utils.cli import ( ADDRESS_TYPE, @@ -37,7 +39,6 @@ option, option_group, ) -from raiden.utils.http import HTTPExecutor from raiden_contracts.constants import CONTRACT_ENDPOINT_REGISTRY, CONTRACT_TOKEN_NETWORK_REGISTRY from .app import run_app @@ -462,14 +463,8 @@ def version(short, **kwargs): # pylint: disable=unused-argument is_flag=True, help='Drop into pdb on errors.', ) -@option( - '--local-matrix', - help='Command-line to be used to run a local matrix server (or "none")', - default=str(Path(__file__).parent.parent.parent.joinpath('.synapse', 'run_synapse.sh')), - show_default=True, -) @click.pass_context -def smoketest(ctx, debug, local_matrix, **kwargs): # pylint: disable=unused-argument +def smoketest(ctx, debug, **kwargs): # pylint: disable=unused-argument """ Test, that the raiden installation is sane. """ from raiden.api.python import RaidenAPI from raiden.tests.utils.smoketest import ( @@ -548,50 +543,53 @@ def _run_smoketest(): del args['extra_config'] args['config'] = config - # invoke the raiden app - app = run_app(**args) - - raiden_api = RaidenAPI(app.raiden) - rest_api = RestAPI(raiden_api) - api_server = APIServer(rest_api) - (api_host, api_port) = split_endpoint(args['api_address']) - api_server.start(api_host, api_port) - - raiden_api.channel_open( - registry_address=contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY], - token_address=to_canonical_address(token.contract.address), - partner_address=to_canonical_address(TEST_PARTNER_ADDRESS), - ) - raiden_api.set_total_channel_deposit( - contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY], - to_canonical_address(token.contract.address), - to_canonical_address(TEST_PARTNER_ADDRESS), - TEST_DEPOSIT_AMOUNT, - ) - token_addresses = [to_checksum_address(token.contract.address)] - - success = False - try: - print_step('Running smoketest') - error = run_smoketests( - app.raiden, - args['transport'], - token_addresses, - contract_addresses[CONTRACT_ENDPOINT_REGISTRY], - debug=debug, + raiden_stdout = StringIO() + with contextlib.redirect_stdout(raiden_stdout): + # invoke the raiden app + app = run_app(**args) + + raiden_api = RaidenAPI(app.raiden) + rest_api = RestAPI(raiden_api) + api_server = APIServer(rest_api) + (api_host, api_port) = split_endpoint(args['api_address']) + api_server.start(api_host, api_port) + + raiden_api.channel_open( + registry_address=contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY], + token_address=to_canonical_address(token.contract.address), + partner_address=to_canonical_address(TEST_PARTNER_ADDRESS), + ) + raiden_api.set_total_channel_deposit( + contract_addresses[CONTRACT_TOKEN_NETWORK_REGISTRY], + to_canonical_address(token.contract.address), + to_canonical_address(TEST_PARTNER_ADDRESS), + TEST_DEPOSIT_AMOUNT, ) - if error is not None: - append_report('Smoketest assertion error', error) - else: - success = True - finally: - app.stop() - node = ethereum[0] - node.send_signal(2) - err, out = node.communicate() - - append_report('Ethereum stdout', out) - append_report('Ethereum stderr', err) + token_addresses = [to_checksum_address(token.contract.address)] + + success = False + try: + print_step('Running smoketest') + error = run_smoketests( + app.raiden, + args['transport'], + token_addresses, + contract_addresses[CONTRACT_ENDPOINT_REGISTRY], + debug=debug, + ) + if error is not None: + append_report('Smoketest assertion error', error) + else: + success = True + finally: + app.stop() + node = ethereum[0] + node.send_signal(2) + err, out = node.communicate() + + append_report('Ethereum stdout', out) + append_report('Ethereum stderr', err) + append_report('Raiden Node stdout', raiden_stdout.getvalue()) if success: print_step(f'Smoketest successful') else: @@ -602,23 +600,18 @@ def _run_smoketest(): with SocketFactory('127.0.0.1', port, strategy='none') as mapped_socket: args['mapped_socket'] = mapped_socket success = _run_smoketest() - elif args['transport'] == 'matrix' and local_matrix.lower() != 'none': + elif args['transport'] == 'matrix': args['mapped_socket'] = None print_step('Starting Matrix transport') try: - with HTTPExecutor( - local_matrix, - url=urljoin(args['matrix_server'], '/_matrix/client/versions'), - method='GET', - io=DEVNULL, - timeout=30, - shell=True, - ): + with matrix_server_starter() as server_urls: + # Disable TLS verification so we can connect to the self signed certificate + make_requests_insecure() + urllib3.disable_warnings(InsecureRequestWarning) args['extra_config'] = { 'transport': { 'matrix': { - 'server_name': 'matrix.local.raiden', - 'available_servers': [], + 'available_servers': server_urls, }, }, } @@ -630,17 +623,6 @@ def _run_smoketest(): error=True, ) success = False - elif args['transport'] == 'matrix' and local_matrix.lower() == "none": - args['mapped_socket'] = None - args['extra_config'] = { - 'transport': { - 'matrix': { - 'server_name': 'matrix.local.raiden', - 'available_servers': [], - }, - }, - } - success = _run_smoketest() else: # Shouldn't happen raise RuntimeError(f"Invalid transport type '{args['transport']}'") diff --git a/raiden/utils/http.py b/raiden/utils/http.py index 83b385414c..a9a418b556 100644 --- a/raiden/utils/http.py +++ b/raiden/utils/http.py @@ -1,7 +1,9 @@ import os import platform import socket +import ssl import subprocess +from http.client import HTTPSConnection from mirakuru.base import ENV_UUID from mirakuru.exceptions import AlreadyRunning, ProcessExitedWithError @@ -11,15 +13,25 @@ class HTTPExecutor(MiHTTPExecutor): """ Subclass off mirakuru.HTTPExecutor, which allows other methods than HEAD """ - def __init__(self, *args, method='HEAD', io=None, **kwargs): + def __init__(self, *args, method='HEAD', io=None, cwd=None, **kwargs): super().__init__(*args, **kwargs) self.method = method self.stdio = io + self.cwd = cwd def after_start_check(self): """ Check if defined URL returns expected status to a request. """ try: - conn = HTTPConnection(self.host, self.port) + if self.url.scheme == 'http': + conn = HTTPConnection(self.host, self.port) + elif self.url.scheme == 'https': + conn = HTTPSConnection( + self.host, + self.port, + context=ssl._create_unverified_context(), + ) + else: + raise ValueError(f'Unsupported URL scheme: "{self.url.scheme}"') conn.request(self.method, self.url.path) status = str(conn.getresponse().status) @@ -32,7 +44,8 @@ def after_start_check(self): return False def start(self): - """ Reimplements Executor and SimpleExecutor start by allowing setting stdin/stdout/stderr + """ + Reimplements Executor and SimpleExecutor start to allow setting stdin/stdout/stderr/cwd It may break input/output/communicate, but will ensure child output redirects won't break parent process by filling the PIPE. @@ -60,6 +73,7 @@ def start(self): 'stderr': stderr, 'universal_newlines': True, 'env': env, + 'cwd': self.cwd, } if platform.system() != 'Windows': popen_kwargs['preexec_fn'] = os.setsid diff --git a/requirements-dev.txt b/requirements-dev.txt index 91831d5640..58a9ad1e57 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -31,3 +31,6 @@ releases==1.6.1 # Release bump2version==0.5.8 + +# Test support +matrix-synapse==0.33.9 diff --git a/setup.cfg b/setup.cfg index 4f94fe7f27..8fe69f6685 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,3 +27,13 @@ omit = */.pyenv/* */tests/* */site-packages/* + +[tool:pytest] +timeout = 540 +norecursedirs = node_modules +; Ignore warnings: +; - grequests monkeypatch +; - urllib3 unverified TLS connection +filterwarnings = + ignore::gevent.monkey.MonkeyPatchWarning + ignore::urllib3.exceptions.InsecureRequestWarning diff --git a/tools/install_synapse.sh b/tools/install_synapse.sh deleted file mode 100755 index 4c1a4243cf..0000000000 --- a/tools/install_synapse.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env bash - -set -exo pipefail - -PYTHON2_VERSION=$(python2 -c 'import sys; print ".".join(str(v) for v in sys.version_info[:2])' || true) - -if [[ ${PYTHON2_VERSION} != "2.7" ]]; then - echo This script requires Python 2.7 - exit 1 -fi - -SYNAPSE_URL="${SYNAPSE_URL:-https://github.com/matrix-org/synapse/archive/v0.33.7.tar.gz#egg=matrix-synapse}" -SYNAPSE_SERVER_NAME="${SYNAPSE_SERVER_NAME:-matrix.local.raiden}" -BASEDIR=$( readlink -f $( dirname $0 )/.. ) - -if [[ ! -d ${DESTDIR} ]]; then - #after travis is not used anymore the following if can be removed - if [[ -n ${TRAVIS} ]]; then - DESTDIR="${HOME}/.bin" # cached folder - else - DESTDIR="${BASEDIR}/.synapse" - mkdir -p "${DESTDIR}" - fi -fi - -# versioned binary according to this script's last commit -SYNAPSE="${DESTDIR}/synapse.$( git log -n1 --pretty=format:%h -- ${0} )" -SYNAPSE_LINK="${DESTDIR}/synapse" -# build synapse single-file executable -# if file not exist or this script is newer than it -if [[ ! -x ${SYNAPSE} ]]; then - if [[ ! -d ${BUILDDIR} ]]; then - BUILDDIR="$( mktemp -d )" - RMBUILDDIR="1" - fi - pushd "${BUILDDIR}" - - virtualenv -p "$(which python2)" venv - ./venv/bin/pip install --upgrade pip pyinstaller - ./venv/bin/pip install pysaml2==4.6.3 dis3 coincurve pycryptodome future html - ./venv/bin/pip install "${SYNAPSE_URL}" - SITE="$( ./venv/bin/python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())' )" - cp "${BASEDIR}/tools/eth_auth_provider.py2" "${SITE}/eth_auth_provider.py" - ./venv/bin/pyinstaller -F -n synapse \ - --hidden-import "sqlite3" \ - --hidden-import "syweb" \ - --hidden-import "eth_auth_provider" \ - --hidden-import "saml2" \ - --hidden-import "html" \ - --add-data="${SITE}/synapse/storage/schema:synapse/storage/schema" \ - --add-data="${SITE}/syweb:syweb" \ - --add-data="${SITE}/Crypto/__init__.py:Crypto/" \ - --add-data="${SITE}/Crypto/Util:Crypto/Util" \ - --add-data="${SITE}/Crypto/Hash:Crypto/Hash" \ - --add-data="${SITE}/pysaml2-4.6.3.dist-info:pysaml2-4.6.3.dist-info" \ - "${SITE}/synapse/app/homeserver.py" - rm -f ${DESTDIR}/synapse.* - cp dist/synapse "${SYNAPSE}" - - popd - [[ -n ${RMBUILDDIR} ]] && rm -r "${BUILDDIR}" -fi - -ln -fs "${SYNAPSE}" "${SYNAPSE_LINK}" -cp "${BASEDIR}/tools/synapse-config.yaml" "${DESTDIR}/" -"${SYNAPSE}" --server-name="${SYNAPSE_SERVER_NAME}" \ - --config-path="${DESTDIR}/synapse-config.yaml" \ - --generate-keys - -cat > "${DESTDIR}/run_synapse.sh" << EOF -#!/usr/bin/env bash -SYNAPSEDIR=\$( dirname "\$0" ) -# redirect synapse stderr logs to stdout -if [[ -n "\${STDOUT_SYNAPSE}" ]]; then - exec 2>&1 -else - mv -vf \${SYNAPSEDIR}/homeserver.log{,.1} - exec &> \${SYNAPSEDIR}/homeserver.log -fi -exec "\${SYNAPSEDIR}/synapse" \ - --server-name="\${SYNAPSE_SERVER_NAME:-${SYNAPSE_SERVER_NAME}}" \ - --config-path="\${SYNAPSEDIR}/synapse-config.yaml" \$@ -EOF -chmod 755 "${DESTDIR}/run_synapse.sh" diff --git a/tox.ini b/tox.ini index d54c0c7c0a..8a76b9a738 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,6 @@ [tox] envlist = coverage -[pytest] -timeout = 540 -norecursedirs = node_modules - [testenv] deps = -rrequirements-dev.txt