From e3e159da068d8be39814156f97438b841e7350a5 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 24 Jan 2019 21:59:35 +1100 Subject: [PATCH 01/43] fix typo --- synapse/storage/_base.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 254fdc04c650..6cc92cceccec 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -221,7 +221,7 @@ def _check_safe_to_upsert(self): # The User IPs table in schema #53 was missing a unique index, which we # run as a background update. if "user_ips_device_unique_index" not in updates: - self._unsafe_to_upsert_tables.discard("user_id") + self._unsafe_to_upsert_tables.discard("user_ips") # If there's any tables left to check, reschedule to run. if self._unsafe_to_upsert_tables: @@ -609,7 +609,7 @@ def _simple_upsert_txn( inserting lock (bool): True to lock the table when doing the upsert. Returns: - Deferred(None or bool): Native upserts always return None. Emulated + None or bool: Native upserts always return None. Emulated upserts return True if a new entry was created, False if an existing one was updated. """ @@ -637,6 +637,18 @@ def _simple_upsert_txn( def _simple_upsert_txn_emulated( self, txn, table, keyvalues, values, insertion_values={}, lock=True ): + """ + Args: + table (str): The table to upsert into + keyvalues (dict): The unique key tables and their new values + values (dict): The nonunique columns and their new values + insertion_values (dict): additional key/values to use only when + inserting + lock (bool): True to lock the table when doing the upsert. + Returns: + bool: Return True if a new entry was created, False if an existing + one was updated. + """ # We need to lock the table :(, unless we're *really* careful if lock: self.database_engine.lock_table(txn, table) From f6a71490cc6d06598070908d6dfec866711160de Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 24 Jan 2019 22:07:02 +1100 Subject: [PATCH 02/43] fix --- changelog.d/4459.misc | 1 + synapse/storage/_base.py | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 changelog.d/4459.misc diff --git a/changelog.d/4459.misc b/changelog.d/4459.misc new file mode 100644 index 000000000000..58130b61908b --- /dev/null +++ b/changelog.d/4459.misc @@ -0,0 +1 @@ +Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+ and SQLite 3.24+. diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 6cc92cceccec..a437a4dd4561 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -29,6 +29,7 @@ from synapse.storage.engines import PostgresEngine from synapse.util.caches.descriptors import Cache from synapse.util.logcontext import LoggingContext, PreserveLoggingContext +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.util.stringutils import exception_to_unicode logger = logging.getLogger(__name__) @@ -198,7 +199,12 @@ def __init__(self, db_conn, hs): if self.database_engine.can_native_upsert: # Check ASAP (and then later, every 1s) to see if we have finished # background updates of tables that aren't safe to update. - self._clock.call_later(0.0, self._check_safe_to_upsert) + self._clock.call_later( + 0.0, + run_as_background_process, + "upsert_safety_check", + self._check_safe_to_upsert + ) @defer.inlineCallbacks def _check_safe_to_upsert(self): @@ -208,7 +214,7 @@ def _check_safe_to_upsert(self): If there are background updates, we will need to wait, as they may be the addition of indexes that set the UNIQUE constraint that we require. - If the background updates have not completed, wait a second and check again. + If the background updates have not completed, wait 15 sec and check again. """ updates = yield self._simple_select_list( "background_updates", @@ -225,7 +231,12 @@ def _check_safe_to_upsert(self): # If there's any tables left to check, reschedule to run. if self._unsafe_to_upsert_tables: - self._clock.call_later(1.0, self._check_safe_to_upsert) + self._clock.call_later( + 15.0, + run_as_background_process, + "upsert_safety_check", + self._check_safe_to_upsert + ) def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() From e148691cd1f280e9db4802cc2e8a0264f8a0dca1 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 24 Jan 2019 22:28:47 +1100 Subject: [PATCH 03/43] fix --- synapse/storage/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index a437a4dd4561..f62f70b9f11f 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -26,10 +26,10 @@ from twisted.internet import defer from synapse.api.errors import StoreError +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.engines import PostgresEngine from synapse.util.caches.descriptors import Cache from synapse.util.logcontext import LoggingContext, PreserveLoggingContext -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.util.stringutils import exception_to_unicode logger = logging.getLogger(__name__) From 3ca24d57eb0d59bd201c583c98fc949b417dca29 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 24 Jan 2019 23:47:33 +1100 Subject: [PATCH 04/43] fix --- synapse/app/homeserver.py | 81 +++++++++++++++++++++++++++++++++++++-- synapse/config/logger.py | 19 +++++---- 2 files changed, 88 insertions(+), 12 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ffc49d77cc36..d1cf4febc5c4 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -17,6 +17,7 @@ import gc import logging import os +import signal import sys import traceback @@ -84,6 +85,7 @@ def gz_wrap(r): class SynapseHomeServer(HomeServer): DATASTORE_CLASS = DataStore + _listening_services = [] def _listener_http(self, config, listener_config): port = listener_config["port"] @@ -121,7 +123,7 @@ def _listener_http(self, config, listener_config): root_resource = create_resource_tree(resources, root_resource) if tls: - listen_ssl( + r = listen_ssl( bind_addresses, port, SynapseSite( @@ -135,7 +137,7 @@ def _listener_http(self, config, listener_config): ) else: - listen_tcp( + r = listen_tcp( bind_addresses, port, SynapseSite( @@ -147,6 +149,7 @@ def _listener_http(self, config, listener_config): ) ) logger.info("Synapse now listening on port %d", port) + return r def _configure_named_resource(self, name, compress=False): """Build a resource map for a named resource @@ -237,12 +240,44 @@ def _configure_named_resource(self, name, compress=False): return resources + def _listen_web(self): + """ + Listen with all the HTTP services. + """ + config = self.get_config() + + for listener in config.listeners: + if listener["type"] == "http": + self._listening_services.append( + self._listener_http(config, listener) + ) + + def _stop_listening_web(self): + """ + Stop listening on all the web services. + + Returns: + Deferred + """ + waiting_on = [] + + for i in self._listening_services: + d = i.stopListening() + if isinstance(d, defer.Deferred): + waiting_on.append(d) + + if waiting_on: + return defer.DeferredList(waiting_on) + else: + return defer.succeed(True) + def start_listening(self): config = self.get_config() for listener in config.listeners: if listener["type"] == "http": - self._listener_http(config, listener) + # Not handled here + pass elif listener["type"] == "manhole": listen_tcp( listener["bind_addresses"], @@ -273,6 +308,9 @@ def start_listening(self): else: logger.warn("Unrecognized listener type: %s", listener["type"]) + # Listen on the web ports differently. + self._listen_web() + def run_startup_checks(self, db_conn, database_engine): all_users_native = are_all_users_on_domain( db_conn.cursor(), database_engine, self.hostname @@ -322,7 +360,19 @@ def setup(config_options): # generating config files and shouldn't try to continue. sys.exit(0) - synapse.config.logger.setup_logging(config, use_worker_options=False) + sighup_callbacks = [] + synapse.config.logger.setup_logging( + config, + use_worker_options=False, + register_sighup=sighup_callbacks.append + ) + + def handle_sighup(*args, **kwargs): + for i in sighup_callbacks: + i(*args, **kwargs) + + if hasattr(signal, "SIGHUP"): + signal.signal(signal.SIGHUP, handle_sighup) events.USE_FROZEN_DICTS = config.use_frozen_dicts @@ -359,6 +409,29 @@ def setup(config_options): hs.setup() + def refresh_certificate(*args): + + logging.info("Stopping web listeners...") + d = hs._stop_listening_web() + + def after(_): + logging.info("Web listeners stopped.") + + logging.info("Reloading certificate from disk") + hs.config.read_certificate_from_disk() + hs.tls_server_context_factory = context_factory.ServerContextFactory(config) + hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( + config + ) + logging.info("Certificate reloaded.") + + logging.info("Restarting web listeners...") + hs._listen_web() + logging.info("Web listeners started.") + d.append(after) + + sighup_callbacks.append(refresh_certificate) + @defer.inlineCallbacks def start(): try: diff --git a/synapse/config/logger.py b/synapse/config/logger.py index f87efecbf8f6..a795e39b1acc 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -127,7 +127,7 @@ def generate_files(self, config): ) -def setup_logging(config, use_worker_options=False): +def setup_logging(config, use_worker_options=False, register_sighup=None): """ Set up python logging Args: @@ -136,7 +136,16 @@ def setup_logging(config, use_worker_options=False): use_worker_options (bool): True to use 'worker_log_config' and 'worker_log_file' options instead of 'log_config' and 'log_file'. + + register_sighup (func | None): Function to call to register a + sighup handler. """ + if not register_sighup: + if getattr(signal, "SIGHUP"): + register_sighup = lambda x: signal.signal(signal.SIGHUP, x) + else: + register_sighup = lambda x: None + log_config = (config.worker_log_config if use_worker_options else config.log_config) log_file = (config.worker_log_file if use_worker_options @@ -198,13 +207,7 @@ def sighup(signum, stack): load_log_config() - # TODO(paul): obviously this is a terrible mechanism for - # stealing SIGHUP, because it means no other part of synapse - # can use it instead. If we want to catch SIGHUP anywhere - # else as well, I'd suggest we find a nicer way to broadcast - # it around. - if getattr(signal, "SIGHUP"): - signal.signal(signal.SIGHUP, sighup) + register_sighup(sighup) # make sure that the first thing we log is a thing we can grep backwards # for From 8aa6d713351e272d9fc98ebfe4cc669b95fd13d0 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 24 Jan 2019 23:47:59 +1100 Subject: [PATCH 05/43] fix --- synapse/app/homeserver.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index d1cf4febc5c4..d2b353d84368 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -409,26 +409,24 @@ def handle_sighup(*args, **kwargs): hs.setup() + @defer.inlineCallbacks def refresh_certificate(*args): logging.info("Stopping web listeners...") - d = hs._stop_listening_web() - - def after(_): - logging.info("Web listeners stopped.") - - logging.info("Reloading certificate from disk") - hs.config.read_certificate_from_disk() - hs.tls_server_context_factory = context_factory.ServerContextFactory(config) - hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - logging.info("Certificate reloaded.") + yield hs._stop_listening_web() + logging.info("Web listeners stopped.") + + logging.info("Reloading certificate from disk") + hs.config.read_certificate_from_disk() + hs.tls_server_context_factory = context_factory.ServerContextFactory(config) + hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( + config + ) + logging.info("Certificate reloaded.") - logging.info("Restarting web listeners...") - hs._listen_web() - logging.info("Web listeners started.") - d.append(after) + logging.info("Restarting web listeners...") + hs._listen_web() + logging.info("Web listeners started.") sighup_callbacks.append(refresh_certificate) From 7ea34b0b202cf1227b5d937ac51be98c691ca2e8 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 24 Jan 2019 23:52:29 +1100 Subject: [PATCH 06/43] fix --- synapse/app/homeserver.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index d2b353d84368..fe978259260d 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -266,6 +266,10 @@ def _stop_listening_web(self): if isinstance(d, defer.Deferred): waiting_on.append(d) + print(waiting_on) + + self._listening_services = [] + if waiting_on: return defer.DeferredList(waiting_on) else: From 70170e09ad2226a98ec9df6105633772441fccb0 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 24 Jan 2019 23:55:12 +1100 Subject: [PATCH 07/43] fix --- synapse/app/homeserver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index fe978259260d..ae7c90604049 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -260,6 +260,7 @@ def _stop_listening_web(self): Deferred """ waiting_on = [] + print(self._listening_services) for i in self._listening_services: d = i.stopListening() From 92e893ef2eac6c500115351b0ce9ebc9cdfb1cb5 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 24 Jan 2019 23:57:06 +1100 Subject: [PATCH 08/43] fix --- synapse/app/homeserver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ae7c90604049..353a57a01ec9 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -373,6 +373,7 @@ def setup(config_options): ) def handle_sighup(*args, **kwargs): + print(sighup_callbacks) for i in sighup_callbacks: i(*args, **kwargs) @@ -415,7 +416,7 @@ def handle_sighup(*args, **kwargs): hs.setup() @defer.inlineCallbacks - def refresh_certificate(*args): + def refresh_certificate(*args, **kwargs): logging.info("Stopping web listeners...") yield hs._stop_listening_web() From f8c4258b720c4373a583cdcbb0e2cea3a0aa0863 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 24 Jan 2019 23:59:02 +1100 Subject: [PATCH 09/43] fix --- synapse/app/homeserver.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 353a57a01ec9..f3fd5ff154f3 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -415,24 +415,27 @@ def handle_sighup(*args, **kwargs): hs.setup() - @defer.inlineCallbacks def refresh_certificate(*args, **kwargs): logging.info("Stopping web listeners...") - yield hs._stop_listening_web() - logging.info("Web listeners stopped.") - - logging.info("Reloading certificate from disk") - hs.config.read_certificate_from_disk() - hs.tls_server_context_factory = context_factory.ServerContextFactory(config) - hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - logging.info("Certificate reloaded.") + d = hs._stop_listening_web() + + def after(_): + logging.info("Web listeners stopped.") + + logging.info("Reloading certificate from disk") + hs.config.read_certificate_from_disk() + hs.tls_server_context_factory = context_factory.ServerContextFactory(config) + hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( + config + ) + logging.info("Certificate reloaded.") + + logging.info("Restarting web listeners...") + hs._listen_web() + logging.info("Web listeners started.") - logging.info("Restarting web listeners...") - hs._listen_web() - logging.info("Web listeners started.") + d.addCallback(after) sighup_callbacks.append(refresh_certificate) From 9a66c51890ecd6f80d822d9f435e61d745cd4e8b Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 24 Jan 2019 23:59:49 +1100 Subject: [PATCH 10/43] fix --- synapse/app/homeserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index f3fd5ff154f3..a8def2e98bda 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -415,7 +415,7 @@ def handle_sighup(*args, **kwargs): hs.setup() - def refresh_certificate(*args, **kwargs): + def refresh_certificate(): logging.info("Stopping web listeners...") d = hs._stop_listening_web() @@ -437,7 +437,7 @@ def after(_): d.addCallback(after) - sighup_callbacks.append(refresh_certificate) + sighup_callbacks.append(lambda *a, **b: reactor.callLater(0.0, refresh_certificate)) @defer.inlineCallbacks def start(): From 7c99df823df90fab7ca2188667faf70ae7285690 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 25 Jan 2019 00:01:53 +1100 Subject: [PATCH 11/43] fix --- synapse/app/homeserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index a8def2e98bda..acd9f9e28fa8 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -260,14 +260,14 @@ def _stop_listening_web(self): Deferred """ waiting_on = [] - print(self._listening_services) + logging.info(self._listening_services) for i in self._listening_services: d = i.stopListening() if isinstance(d, defer.Deferred): waiting_on.append(d) - print(waiting_on) + logging.info(repr(waiting_on)) self._listening_services = [] From 0f0187e8cb8dcc2496bb85e07be20e5bbefce654 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 25 Jan 2019 00:03:10 +1100 Subject: [PATCH 12/43] fix --- synapse/util/async_helpers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 430bb15f51da..f0e4a0e10c1d 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -201,7 +201,7 @@ def queue(self, key): if entry[0] >= self.max_count: res = self._await_lock(key) else: - logger.info( + logger.debug( "Acquired uncontended linearizer lock %r for key %r", self.name, key, ) entry[0] += 1 @@ -215,7 +215,7 @@ def _ctx_manager(_): try: yield finally: - logger.info("Releasing linearizer lock %r for key %r", self.name, key) + logger.debug("Releasing linearizer lock %r for key %r", self.name, key) # We've finished executing so check if there are any things # blocked waiting to execute and start one of them @@ -247,7 +247,7 @@ def _await_lock(self, key): """ entry = self.key_to_defer[key] - logger.info( + logger.debug( "Waiting to acquire linearizer lock %r for key %r", self.name, key, ) @@ -255,7 +255,7 @@ def _await_lock(self, key): entry[1][new_defer] = 1 def cb(_r): - logger.info("Acquired linearizer lock %r for key %r", self.name, key) + logger.debug("Acquired linearizer lock %r for key %r", self.name, key) entry[0] += 1 # if the code holding the lock completes synchronously, then it @@ -273,7 +273,7 @@ def cb(_r): def eb(e): logger.info("defer %r got err %r", new_defer, e) if isinstance(e, CancelledError): - logger.info( + logger.debug( "Cancelling wait for linearizer lock %r for key %r", self.name, key, ) From 4b9fd2be35a070a2a3df2dd19e2adbde5ed36040 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 25 Jan 2019 00:05:21 +1100 Subject: [PATCH 13/43] fix --- synapse/app/_base.py | 19 +++++++++++++------ synapse/app/homeserver.py | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 18584226e9c4..3749a15f19d9 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -155,6 +155,8 @@ def listen_tcp(bind_addresses, port, factory, reactor=reactor, backlog=50): except error.CannotListenError as e: check_bind_error(e, address, bind_addresses) + return [] + def listen_ssl( bind_addresses, port, factory, context_factory, reactor=reactor, backlog=50 @@ -162,18 +164,23 @@ def listen_ssl( """ Create an SSL socket for a port and several addresses """ + r = [] for address in bind_addresses: try: - reactor.listenSSL( - port, - factory, - context_factory, - backlog, - address + r.append( + reactor.listenSSL( + port, + factory, + context_factory, + backlog, + address + ) ) except error.CannotListenError as e: check_bind_error(e, address, bind_addresses) + return r + def check_bind_error(e, address, bind_addresses): """ diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index acd9f9e28fa8..fc4ade9278cc 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -248,7 +248,7 @@ def _listen_web(self): for listener in config.listeners: if listener["type"] == "http": - self._listening_services.append( + self._listening_services.extend( self._listener_http(config, listener) ) From e2615be4244faad35d535e46df9ecb8f85295d34 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 25 Jan 2019 00:12:49 +1100 Subject: [PATCH 14/43] fix --- synapse/app/_base.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 3749a15f19d9..18fc09a9b9b3 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -144,18 +144,21 @@ def listen_tcp(bind_addresses, port, factory, reactor=reactor, backlog=50): """ Create a TCP socket for a port and several addresses """ + r = [] for address in bind_addresses: try: - reactor.listenTCP( - port, - factory, - backlog, - address + r.append( + reactor.listenTCP( + port, + factory, + backlog, + address + ) ) except error.CannotListenError as e: check_bind_error(e, address, bind_addresses) - return [] + return r def listen_ssl( From abc4e7d0126b5cead65144b13a920ddae6b87f8e Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 25 Jan 2019 00:25:42 +1100 Subject: [PATCH 15/43] fix --- synapse/app/homeserver.py | 75 ++++++++------------------------------- 1 file changed, 15 insertions(+), 60 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index fc4ade9278cc..a97a9e44a901 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -123,7 +123,7 @@ def _listener_http(self, config, listener_config): root_resource = create_resource_tree(resources, root_resource) if tls: - r = listen_ssl( + listen_ssl( bind_addresses, port, SynapseSite( @@ -240,10 +240,7 @@ def _configure_named_resource(self, name, compress=False): return resources - def _listen_web(self): - """ - Listen with all the HTTP services. - """ + def start_listening(self): config = self.get_config() for listener in config.listeners: @@ -251,38 +248,6 @@ def _listen_web(self): self._listening_services.extend( self._listener_http(config, listener) ) - - def _stop_listening_web(self): - """ - Stop listening on all the web services. - - Returns: - Deferred - """ - waiting_on = [] - logging.info(self._listening_services) - - for i in self._listening_services: - d = i.stopListening() - if isinstance(d, defer.Deferred): - waiting_on.append(d) - - logging.info(repr(waiting_on)) - - self._listening_services = [] - - if waiting_on: - return defer.DeferredList(waiting_on) - else: - return defer.succeed(True) - - def start_listening(self): - config = self.get_config() - - for listener in config.listeners: - if listener["type"] == "http": - # Not handled here - pass elif listener["type"] == "manhole": listen_tcp( listener["bind_addresses"], @@ -313,9 +278,6 @@ def start_listening(self): else: logger.warn("Unrecognized listener type: %s", listener["type"]) - # Listen on the web ports differently. - self._listen_web() - def run_startup_checks(self, db_conn, database_engine): all_users_native = are_all_users_on_domain( db_conn.cursor(), database_engine, self.hostname @@ -415,29 +377,22 @@ def handle_sighup(*args, **kwargs): hs.setup() - def refresh_certificate(): - - logging.info("Stopping web listeners...") - d = hs._stop_listening_web() - - def after(_): - logging.info("Web listeners stopped.") + def refresh_certificate(*args): - logging.info("Reloading certificate from disk") - hs.config.read_certificate_from_disk() - hs.tls_server_context_factory = context_factory.ServerContextFactory(config) - hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - logging.info("Certificate reloaded.") - - logging.info("Restarting web listeners...") - hs._listen_web() - logging.info("Web listeners started.") + logging.info("Reloading certificate from disk") + hs.config.read_certificate_from_disk() + hs.tls_server_context_factory = context_factory.ServerContextFactory(config) + hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( + config + ) + logging.info("Certificate reloaded.") - d.addCallback(after) + logging.info("Updating context factories...") + for i in hs._listening_services: + i.ctxFactory = hs.tls_server_context_factory + logging.info("Context factories updated..") - sighup_callbacks.append(lambda *a, **b: reactor.callLater(0.0, refresh_certificate)) + sighup_callbacks.append(refresh_certificate) @defer.inlineCallbacks def start(): From 666fc907ea1f51127e5092fd4d853f2cb6965cf6 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 25 Jan 2019 00:39:10 +1100 Subject: [PATCH 16/43] fix --- synapse/app/homeserver.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index a97a9e44a901..19f982c0fdc0 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -123,7 +123,7 @@ def _listener_http(self, config, listener_config): root_resource = create_resource_tree(resources, root_resource) if tls: - listen_ssl( + r = listen_ssl( bind_addresses, port, SynapseSite( @@ -335,7 +335,6 @@ def setup(config_options): ) def handle_sighup(*args, **kwargs): - print(sighup_callbacks) for i in sighup_callbacks: i(*args, **kwargs) From f6b58aa312acdf40ee3e9eea1abed766b2e19460 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 25 Jan 2019 00:45:28 +1100 Subject: [PATCH 17/43] fix --- synapse/app/homeserver.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 19f982c0fdc0..90e7683d465f 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -26,6 +26,8 @@ import psutil from prometheus_client import Gauge + +from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.application import service from twisted.internet import defer, reactor from twisted.web.resource import EncodingResourceWrapper, NoResource @@ -388,8 +390,14 @@ def refresh_certificate(*args): logging.info("Updating context factories...") for i in hs._listening_services: - i.ctxFactory = hs.tls_server_context_factory - logging.info("Context factories updated..") + if isinstance(i.factory, TLSMemoryBIOFactory): + old = i.factory + i.factory = TLSMemoryBIOFactory( + hs.tls_server_context_factory, + False, + i.factory.wrappedFactory + ) + logging.info("Context factories updated.") sighup_callbacks.append(refresh_certificate) From aaf9220eaa5082f4592e3abe2aa0c11b82820db7 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 28 Jan 2019 17:25:33 +0000 Subject: [PATCH 18/43] fix --- synapse/storage/_base.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 247517f07493..5109bc3e2e2e 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -27,11 +27,7 @@ from synapse.api.errors import StoreError from synapse.metrics.background_process_metrics import run_as_background_process -<<<<<<< HEAD -from synapse.storage.engines import PostgresEngine -======= from synapse.storage.engines import PostgresEngine, Sqlite3Engine ->>>>>>> origin/develop from synapse.util.caches.descriptors import Cache from synapse.util.logcontext import LoggingContext, PreserveLoggingContext from synapse.util.stringutils import exception_to_unicode From 7df3114e7e4cffc8c2429dc84f9388c5f8feb656 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 28 Jan 2019 17:26:09 +0000 Subject: [PATCH 19/43] changelog --- changelog.d/4495.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4495.feature diff --git a/changelog.d/4495.feature b/changelog.d/4495.feature new file mode 100644 index 000000000000..fc2b5daf637e --- /dev/null +++ b/changelog.d/4495.feature @@ -0,0 +1 @@ +Synapse will now reload TLS certificates from disk upon SIGHUP. From 3fe07f786e65ca79c04440e71de7308b7d9a6d2f Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 29 Jan 2019 08:39:39 +0000 Subject: [PATCH 20/43] fix --- synapse/app/homeserver.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 90e7683d465f..4c176c48abeb 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -26,10 +26,9 @@ import psutil from prometheus_client import Gauge - -from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.application import service from twisted.internet import defer, reactor +from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.web.resource import EncodingResourceWrapper, NoResource from twisted.web.server import GzipEncoderFactory from twisted.web.static import File From 2340ae52447f7b8e7839106dd2492a50742e1928 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 29 Jan 2019 08:40:18 +0000 Subject: [PATCH 21/43] fix --- synapse/app/homeserver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 4c176c48abeb..90a25d5d95e3 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -390,7 +390,6 @@ def refresh_certificate(*args): logging.info("Updating context factories...") for i in hs._listening_services: if isinstance(i.factory, TLSMemoryBIOFactory): - old = i.factory i.factory = TLSMemoryBIOFactory( hs.tls_server_context_factory, False, From 08de6c9139710b071ed8ad50c8f79ca9ea30d94d Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 29 Jan 2019 08:53:41 +0000 Subject: [PATCH 22/43] fix --- synapse/storage/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 5109bc3e2e2e..4872ff55b634 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -236,7 +236,7 @@ def _check_safe_to_upsert(self): self._unsafe_to_upsert_tables.discard("user_ips") # If there's any tables left to check, reschedule to run. - if self.updates: + if updates: self._clock.call_later( 15.0, run_as_background_process, From 14e4c4f67f70d41b69f945b5ee683856d400df65 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 29 Jan 2019 14:01:01 +0000 Subject: [PATCH 23/43] fix --- .gitignore | 1 + synapse/app/_base.py | 25 +++++++++++++++---------- synapse/app/homeserver.py | 15 +++++++++------ 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 1033124f1d26..37f0028d66dc 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ dbs/ dist/ docs/build/ *.egg-info +pip-wheel-metadata/ cmdclient_config.json homeserver*.db diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 18fc09a9b9b3..3840c663abd8 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -143,29 +143,33 @@ def listen_metrics(bind_addresses, port): def listen_tcp(bind_addresses, port, factory, reactor=reactor, backlog=50): """ Create a TCP socket for a port and several addresses + + Returns: + list (empty) """ - r = [] for address in bind_addresses: try: - r.append( - reactor.listenTCP( - port, - factory, - backlog, - address - ) + reactor.listenTCP( + port, + factory, + backlog, + address ) except error.CannotListenError as e: check_bind_error(e, address, bind_addresses) - return r + logger.info("Synapse now listening on TCP port %d", port) + return [] def listen_ssl( bind_addresses, port, factory, context_factory, reactor=reactor, backlog=50 ): """ - Create an SSL socket for a port and several addresses + Create an TLS-over-TCP socket for a port and several addresses + + Returns: + list of twisted.internet.tcp.Port listening for TLS connections """ r = [] for address in bind_addresses: @@ -182,6 +186,7 @@ def listen_ssl( except error.CannotListenError as e: check_bind_error(e, address, bind_addresses) + logger.info("Synapse now listening on port %d (TLS)", port) return r diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 90a25d5d95e3..b5442d43ae3b 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -123,8 +123,10 @@ def _listener_http(self, config, listener_config): root_resource = create_resource_tree(resources, root_resource) + port = None + if tls: - r = listen_ssl( + return listen_ssl( bind_addresses, port, SynapseSite( @@ -138,7 +140,7 @@ def _listener_http(self, config, listener_config): ) else: - r = listen_tcp( + return listen_tcp( bind_addresses, port, SynapseSite( @@ -149,8 +151,6 @@ def _listener_http(self, config, listener_config): self.version_string, ) ) - logger.info("Synapse now listening on port %d", port) - return r def _configure_named_resource(self, name, compress=False): """Build a resource map for a named resource @@ -378,8 +378,11 @@ def handle_sighup(*args, **kwargs): hs.setup() def refresh_certificate(*args): - - logging.info("Reloading certificate from disk") + """ + Refresh the TLS certificates that Synapse is using by re-reading them + from disk and updating the TLS context factories to use them. + """ + logging.info("Reloading certificate from disk...") hs.config.read_certificate_from_disk() hs.tls_server_context_factory = context_factory.ServerContextFactory(config) hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( From d08ef7be0f93f0ac9d2a04d88a14e05f5e7a8081 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 29 Jan 2019 14:11:57 +0000 Subject: [PATCH 24/43] fix --- synapse/app/homeserver.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index b5442d43ae3b..019b91576fbf 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -123,8 +123,6 @@ def _listener_http(self, config, listener_config): root_resource = create_resource_tree(resources, root_resource) - port = None - if tls: return listen_ssl( bind_addresses, From 45f9d6cef1942343d118fc47420f40d2c03192a0 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 29 Jan 2019 16:37:47 +0000 Subject: [PATCH 25/43] reprovisioning code --- synapse/app/homeserver.py | 99 +++++++++++++++++++++++++-------------- synapse/config/tls.py | 12 ++++- 2 files changed, 76 insertions(+), 35 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 019b91576fbf..34f35c0638ad 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -380,56 +380,87 @@ def refresh_certificate(*args): Refresh the TLS certificates that Synapse is using by re-reading them from disk and updating the TLS context factories to use them. """ - logging.info("Reloading certificate from disk...") + logging.info("Loading certificate from disk...") hs.config.read_certificate_from_disk() hs.tls_server_context_factory = context_factory.ServerContextFactory(config) hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( config ) - logging.info("Certificate reloaded.") - - logging.info("Updating context factories...") - for i in hs._listening_services: - if isinstance(i.factory, TLSMemoryBIOFactory): - i.factory = TLSMemoryBIOFactory( - hs.tls_server_context_factory, - False, - i.factory.wrappedFactory - ) - logging.info("Context factories updated.") + logging.info("Certificate loaded.") + + if hs._listening_services: + logging.info("Updating context factories...") + for i in hs._listening_services: + if isinstance(i.factory, TLSMemoryBIOFactory): + i.factory = TLSMemoryBIOFactory( + hs.tls_server_context_factory, + False, + i.factory.wrappedFactory + ) + logging.info("Context factories updated.") sighup_callbacks.append(refresh_certificate) + @defer.inlineCallbacks + def do_acme(): + """ + Reprovision an ACME certificate, if it's required. + + Returns: + Deferred[bool]: Whether the cert has been updated. + """ + acme = hs.get_acme_handler() + + # Check how long the certificate is active for. + cert_days_remaining = hs.config.is_disk_cert_valid( + allow_self_signed=False + ) + + # We want to reprovision if cert_days_remaining is None (meaning no + # certificate exists), or the days remaining number it returns + # is less than our re-registration threshold. + provision = False + + if (cert_days_remaining is None): + provision = True + + if cert_days_remaining > hs.config.acme_reprovision_threshold: + provision = True + + if provision: + yield acme.provision_certificate() + + defer.returnValue(provision) + + @defer.inlineCallbacks + def reprovision_acme(): + """ + Provision a certificate from ACME, if required, and reload the TLS + certificate if it's renewed. + """ + reprovisioned = yield do_acme() + if reprovisioned: + refresh_certificate() + @defer.inlineCallbacks def start(): try: - # Check if the certificate is still valid. - cert_days_remaining = hs.config.is_disk_cert_valid() - + # Run the ACME provisioning code, if it's enabled. if hs.config.acme_enabled: - # If ACME is enabled, we might need to provision a certificate - # before starting. acme = hs.get_acme_handler() - # Start up the webservices which we will respond to ACME - # challenges with. + # challenges with, and then provision. yield acme.start_listening() + yield do_acme() - # We want to reprovision if cert_days_remaining is None (meaning no - # certificate exists), or the days remaining number it returns - # is less than our re-registration threshold. - if (cert_days_remaining is None) or ( - not cert_days_remaining > hs.config.acme_reprovision_threshold - ): - yield acme.provision_certificate() - - # Read the certificate from disk and build the context factories for - # TLS. - hs.config.read_certificate_from_disk() - hs.tls_server_context_factory = context_factory.ServerContextFactory(config) - hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) + # Check if it needs to be reprovisioned every day. + hs.get_clock().looping_call( + reprovision_acme, + 24 * 60 * 60 * 1000 + ) + + # Load the certificate from disk. + refresh_certificate() # It is now safe to start your Synapse. hs.start_listening() diff --git a/synapse/config/tls.py b/synapse/config/tls.py index a75e233aa0ee..d59797a57910 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -56,10 +56,14 @@ def read_config(self, config): self.tls_certificate = None self.tls_private_key = None - def is_disk_cert_valid(self): + def is_disk_cert_valid(self, allow_self_signed=True): """ Is the certificate we have on disk valid, and if so, for how long? + Args: + allow_self_signed (bool): Should we allow the certificate we + read to be self signed? + Returns: int: Days remaining of certificate validity. None: No certificate exists. @@ -80,6 +84,12 @@ def is_disk_cert_valid(self): logger.exception("Failed to parse existing certificate off disk!") raise + if not allow_self_signed: + if tls_certificate.get_subject() == tls_certificate.get_issuer(): + raise ValueError( + "TLS Certificate is self signed, and this is not permitted" + ) + # YYYYMMDDhhmmssZ -- in UTC expires_on = datetime.strptime( tls_certificate.get_notAfter().decode('ascii'), "%Y%m%d%H%M%SZ" From 97eec0a1cf88accb1249f0a35d389d04f52fc5a9 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 30 Jan 2019 10:18:23 +0000 Subject: [PATCH 26/43] reprovisioning code --- synapse/app/homeserver.py | 16 ++++++++-------- synapse/config/logger.py | 9 +++------ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 34f35c0638ad..06603dd9bc05 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -333,13 +333,6 @@ def setup(config_options): register_sighup=sighup_callbacks.append ) - def handle_sighup(*args, **kwargs): - for i in sighup_callbacks: - i(*args, **kwargs) - - if hasattr(signal, "SIGHUP"): - signal.signal(signal.SIGHUP, handle_sighup) - events.USE_FROZEN_DICTS = config.use_frozen_dicts database_engine = create_engine(config.database_config) @@ -353,6 +346,13 @@ def handle_sighup(*args, **kwargs): database_engine=database_engine, ) + def handle_sighup(*args, **kwargs): + for i in sighup_callbacks: + i(hs) + + if hasattr(signal, "SIGHUP"): + signal.signal(signal.SIGHUP, handle_sighup) + logger.info("Preparing database: %s...", config.database_config['name']) try: @@ -375,7 +375,7 @@ def handle_sighup(*args, **kwargs): hs.setup() - def refresh_certificate(*args): + def refresh_certificate(hs): """ Refresh the TLS certificates that Synapse is using by re-reading them from disk and updating the TLS context factories to use them. diff --git a/synapse/config/logger.py b/synapse/config/logger.py index a795e39b1acc..d06224e20c2f 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -141,10 +141,7 @@ def setup_logging(config, use_worker_options=False, register_sighup=None): sighup handler. """ if not register_sighup: - if getattr(signal, "SIGHUP"): - register_sighup = lambda x: signal.signal(signal.SIGHUP, x) - else: - register_sighup = lambda x: None + register_sighup = lambda x: None log_config = (config.worker_log_config if use_worker_options else config.log_config) @@ -187,7 +184,7 @@ def sighup(signum, stack): else: handler = logging.StreamHandler() - def sighup(signum, stack): + def sighup(*args): pass handler.setFormatter(formatter) @@ -200,7 +197,7 @@ def load_log_config(): with open(log_config, 'r') as f: logging.config.dictConfig(yaml.load(f)) - def sighup(signum, stack): + def sighup(*args): # it might be better to use a file watcher or something for this. load_log_config() logging.info("Reloaded log config from %s due to SIGHUP", log_config) From 62b4e0168c5de99381e134ad280b64f7dc6eabf5 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 30 Jan 2019 12:38:15 +0000 Subject: [PATCH 27/43] pep8 fixes --- synapse/app/_base.py | 62 +++++++++++++++++++++++++++++++++++++++ synapse/app/homeserver.py | 55 ++++++---------------------------- synapse/config/logger.py | 9 ++---- 3 files changed, 74 insertions(+), 52 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 3840c663abd8..cbed39d1b9f0 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -15,18 +15,27 @@ import gc import logging +import traceback import sys +import signal import psutil +from synapse.crypto import context_factory + from daemonize import Daemonize from twisted.internet import error, reactor +from twisted.protocols.tls import TLSMemoryBIOFactory + from synapse.util import PreserveLoggingContext from synapse.util.rlimit import change_resource_limit logger = logging.getLogger(__name__) +_sighup_callbacks = [] +register_sighup = _sighup_callbacks.append + def start_worker_reactor(appname, config): """ Run the reactor in the main process @@ -209,3 +218,56 @@ def check_bind_error(e, address, bind_addresses): logger.warn('Failed to listen on 0.0.0.0, continuing because listening on [::]') else: raise e + + +def refresh_certificate(hs): + """ + Refresh the TLS certificates that Synapse is using by re-reading them from + disk and updating the TLS context factories to use them. + """ + logging.info("Loading certificate from disk...") + hs.config.read_certificate_from_disk() + hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config) + hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( + hs.config + ) + logging.info("Certificate loaded.") + + if hs._listening_services: + logging.info("Updating context factories...") + for i in hs._listening_services: + if isinstance(i.factory, TLSMemoryBIOFactory): + i.factory = TLSMemoryBIOFactory( + hs.tls_server_context_factory, + False, + i.factory.wrappedFactory + ) + logging.info("Context factories updated.") + + +def start(hs): + """ + Start a Synapse server or worker. + """ + try: + def handle_sighup(*args, **kwargs): + for i in _sighup_callbacks: + i(hs) + + if hasattr(signal, "SIGHUP"): + signal.signal(signal.SIGHUP, handle_sighup) + + register_sighup(refresh_certificate) + + # Load the certificate from disk. + refresh_certificate(hs) + + # It is now safe to start your Synapse. + hs.start_listening() + hs.get_datastore().start_profiling() + except Exception: + traceback.print_exc(file=sys.stderr) + reactor = hs.get_reactor() + if reactor.running: + reactor.stop() + sys.exit(1) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 06603dd9bc05..bbe709e2898d 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -17,7 +17,6 @@ import gc import logging import os -import signal import sys import traceback @@ -28,7 +27,6 @@ from twisted.application import service from twisted.internet import defer, reactor -from twisted.protocols.tls import TLSMemoryBIOFactory from twisted.web.resource import EncodingResourceWrapper, NoResource from twisted.web.server import GzipEncoderFactory from twisted.web.static import File @@ -49,7 +47,6 @@ from synapse.app._base import listen_ssl, listen_tcp, quit_with_error from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig -from synapse.crypto import context_factory from synapse.federation.transport.server import TransportLayerServer from synapse.http.additional_resource import AdditionalResource from synapse.http.server import RootRedirect @@ -326,11 +323,9 @@ def setup(config_options): # generating config files and shouldn't try to continue. sys.exit(0) - sighup_callbacks = [] synapse.config.logger.setup_logging( config, - use_worker_options=False, - register_sighup=sighup_callbacks.append + use_worker_options=False ) events.USE_FROZEN_DICTS = config.use_frozen_dicts @@ -346,13 +341,6 @@ def setup(config_options): database_engine=database_engine, ) - def handle_sighup(*args, **kwargs): - for i in sighup_callbacks: - i(hs) - - if hasattr(signal, "SIGHUP"): - signal.signal(signal.SIGHUP, handle_sighup) - logger.info("Preparing database: %s...", config.database_config['name']) try: @@ -375,32 +363,6 @@ def handle_sighup(*args, **kwargs): hs.setup() - def refresh_certificate(hs): - """ - Refresh the TLS certificates that Synapse is using by re-reading them - from disk and updating the TLS context factories to use them. - """ - logging.info("Loading certificate from disk...") - hs.config.read_certificate_from_disk() - hs.tls_server_context_factory = context_factory.ServerContextFactory(config) - hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - logging.info("Certificate loaded.") - - if hs._listening_services: - logging.info("Updating context factories...") - for i in hs._listening_services: - if isinstance(i.factory, TLSMemoryBIOFactory): - i.factory = TLSMemoryBIOFactory( - hs.tls_server_context_factory, - False, - i.factory.wrappedFactory - ) - logging.info("Context factories updated.") - - sighup_callbacks.append(refresh_certificate) - @defer.inlineCallbacks def do_acme(): """ @@ -440,7 +402,7 @@ def reprovision_acme(): """ reprovisioned = yield do_acme() if reprovisioned: - refresh_certificate() + _base.refresh_certificate(hs) @defer.inlineCallbacks def start(): @@ -459,23 +421,23 @@ def start(): 24 * 60 * 60 * 1000 ) - # Load the certificate from disk. - refresh_certificate() + _base.start(hs) - # It is now safe to start your Synapse. - hs.start_listening() hs.get_pusherpool().start() - hs.get_datastore().start_profiling() hs.get_datastore().start_doing_background_updates() except Exception as e: # If a DeferredList failed (like in listening on the ACME listener), # we need to print the subfailure explicitly. if isinstance(e, defer.FirstError): e.subFailure.printTraceback(sys.stderr) + if reactor.running: + reactor.stop() sys.exit(1) # Something else went wrong when starting. Print it and bail out. traceback.print_exc(file=sys.stderr) + if reactor.running: + reactor.stop() sys.exit(1) reactor.callWhenRunning(start) @@ -484,7 +446,8 @@ def start(): class SynapseService(service.Service): - """A twisted Service class that will start synapse. Used to run synapse + """ + A twisted Service class that will start synapse. Used to run synapse via twistd and a .tac. """ def __init__(self, config): diff --git a/synapse/config/logger.py b/synapse/config/logger.py index d06224e20c2f..4b938053fb75 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -15,7 +15,6 @@ import logging import logging.config import os -import signal import sys from string import Template @@ -24,6 +23,7 @@ from twisted.logger import STDLibLogObserver, globalLogBeginner import synapse +from synapse.app import _base as appbase from synapse.util.logcontext import LoggingContextFilter from synapse.util.versionstring import get_version_string @@ -127,7 +127,7 @@ def generate_files(self, config): ) -def setup_logging(config, use_worker_options=False, register_sighup=None): +def setup_logging(config, use_worker_options=False): """ Set up python logging Args: @@ -140,9 +140,6 @@ def setup_logging(config, use_worker_options=False, register_sighup=None): register_sighup (func | None): Function to call to register a sighup handler. """ - if not register_sighup: - register_sighup = lambda x: None - log_config = (config.worker_log_config if use_worker_options else config.log_config) log_file = (config.worker_log_file if use_worker_options @@ -204,7 +201,7 @@ def sighup(*args): load_log_config() - register_sighup(sighup) + appbase.register_sighup(sighup) # make sure that the first thing we log is a thing we can grep backwards # for From 965921cc65eaf7a97ea6dbed0e664b61412be9be Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 30 Jan 2019 13:39:47 +0000 Subject: [PATCH 28/43] pep8 fixes --- synapse/app/_base.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index cbed39d1b9f0..e2d3bc7ee79c 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -15,19 +15,17 @@ import gc import logging -import traceback -import sys import signal +import sys +import traceback import psutil -from synapse.crypto import context_factory - from daemonize import Daemonize from twisted.internet import error, reactor - from twisted.protocols.tls import TLSMemoryBIOFactory +from synapse.crypto import context_factory from synapse.util import PreserveLoggingContext from synapse.util.rlimit import change_resource_limit From d21f762bd73be08b349a002c343b2c63ea3882f4 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 30 Jan 2019 14:01:48 +0000 Subject: [PATCH 29/43] fixes --- synapse/app/appservice.py | 7 +------ synapse/app/client_reader.py | 13 +------------ synapse/app/event_creator.py | 13 +------------ synapse/app/federation_reader.py | 13 +------------ synapse/app/federation_sender.py | 12 +----------- synapse/app/frontend_proxy.py | 13 +------------ synapse/app/homeserver.py | 1 - synapse/app/media_repository.py | 13 +------------ synapse/app/pusher.py | 3 +-- synapse/app/synchrotron.py | 7 +------ synapse/app/user_dir.py | 13 +------------ synapse/server.py | 1 + 12 files changed, 11 insertions(+), 98 deletions(-) diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py index 8559e141af47..33107f56d137 100644 --- a/synapse/app/appservice.py +++ b/synapse/app/appservice.py @@ -168,12 +168,7 @@ def start(config_options): ) ps.setup() - ps.start_listening(config.worker_listeners) - - def start(): - ps.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ps, config.worker_listeners) _base.start_worker_reactor("synapse-appservice", config) diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py index f8a417cb6070..a9d214702286 100644 --- a/synapse/app/client_reader.py +++ b/synapse/app/client_reader.py @@ -25,7 +25,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -173,17 +172,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-client-reader", config) diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py index 656e0edc0fbc..b8e51961528b 100644 --- a/synapse/app/event_creator.py +++ b/synapse/app/event_creator.py @@ -25,7 +25,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -194,17 +193,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-event-creator", config) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index 3de27151328c..42886dbfc149 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -26,7 +26,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.federation.transport.server import TransportLayerServer from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -160,17 +159,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-federation-reader", config) diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py index d944e0517f2e..a461442fdc41 100644 --- a/synapse/app/federation_sender.py +++ b/synapse/app/federation_sender.py @@ -25,7 +25,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.federation import send_queue from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -192,17 +191,8 @@ def start(config_options): ) ss.setup() + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) _base.start_worker_reactor("synapse-federation-sender", config) diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py index d9ef6edc3c6d..d5b954361d9a 100644 --- a/synapse/app/frontend_proxy.py +++ b/synapse/app/frontend_proxy.py @@ -26,7 +26,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseSite @@ -250,17 +249,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-frontend-proxy", config) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index bbe709e2898d..a08c27f709e0 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -83,7 +83,6 @@ def gz_wrap(r): class SynapseHomeServer(HomeServer): DATASTORE_CLASS = DataStore - _listening_services = [] def _listener_http(self, config, listener_config): port = listener_config["port"] diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py index 4ecf64031bf7..d4cc4e9443c7 100644 --- a/synapse/app/media_repository.py +++ b/synapse/app/media_repository.py @@ -26,7 +26,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy from synapse.metrics.resource import METRICS_PREFIX, MetricsResource @@ -160,17 +159,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-media-repository", config) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 83b0863f00f4..cbf0d67f51fa 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -224,11 +224,10 @@ def start(config_options): ) ps.setup() - ps.start_listening(config.worker_listeners) def start(): + _base.start(ps, config.worker_listeners) ps.get_pusherpool().start() - ps.get_datastore().start_profiling() reactor.callWhenRunning(start) diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index 0354e82bf850..9163b56d86fb 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -445,12 +445,7 @@ def start(config_options): ) ss.setup() - ss.start_listening(config.worker_listeners) - - def start(): - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-synchrotron", config) diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py index 176d55a78333..d1ab9512cd07 100644 --- a/synapse/app/user_dir.py +++ b/synapse/app/user_dir.py @@ -26,7 +26,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -220,17 +219,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-user-dir", config) diff --git a/synapse/server.py b/synapse/server.py index 6c521016163f..cb35a7bab3cc 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -196,6 +196,7 @@ def __init__(self, hostname, reactor=None, **kwargs): self._reactor = reactor self.hostname = hostname self._building = {} + self._listening_services = [] self.clock = Clock(reactor) self.distributor = Distributor() From 14274ebb74d26efaf2fa845804bcdc342f10e063 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 30 Jan 2019 14:02:59 +0000 Subject: [PATCH 30/43] fixes --- synapse/app/_base.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index e2d3bc7ee79c..5a6932eea2d8 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -243,25 +243,26 @@ def refresh_certificate(hs): logging.info("Context factories updated.") -def start(hs): +def start(hs, listeners=None): """ Start a Synapse server or worker. """ try: - def handle_sighup(*args, **kwargs): - for i in _sighup_callbacks: - i(hs) - + # Set up the SIGHUP machinery. if hasattr(signal, "SIGHUP"): + def handle_sighup(*args, **kwargs): + for i in _sighup_callbacks: + i(hs) + signal.signal(signal.SIGHUP, handle_sighup) - register_sighup(refresh_certificate) + register_sighup(refresh_certificate) # Load the certificate from disk. refresh_certificate(hs) # It is now safe to start your Synapse. - hs.start_listening() + hs.start_listening(listeners) hs.get_datastore().start_profiling() except Exception: traceback.print_exc(file=sys.stderr) From 12696abb484ad1cd724630b4307ba70f9549ec26 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 30 Jan 2019 14:07:57 +0000 Subject: [PATCH 31/43] fixes --- synapse/app/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 8ee9f096e0d1..28877c76cdc1 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -237,7 +237,7 @@ def _configure_named_resource(self, name, compress=False): return resources - def start_listening(self): + def start_listening(self, ign): config = self.get_config() for listener in config.listeners: From fe36f24381f9558d4236ecc26bc5825b6c69b5f7 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 30 Jan 2019 14:10:13 +0000 Subject: [PATCH 32/43] fixes --- synapse/app/homeserver.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 28877c76cdc1..c10baf9378a8 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -426,16 +426,8 @@ def start(): hs.get_pusherpool().start() hs.get_datastore().start_doing_background_updates() - except Exception as e: - # If a DeferredList failed (like in listening on the ACME listener), - # we need to print the subfailure explicitly. - if isinstance(e, defer.FirstError): - e.subFailure.printTraceback(sys.stderr) - if reactor.running: - reactor.stop() - sys.exit(1) - - # Something else went wrong when starting. Print it and bail out. + except Exception: + # Print the exception and bail out. traceback.print_exc(file=sys.stderr) if reactor.running: reactor.stop() From 9c9d261014e67dc317cba49a3f194237905867b4 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 30 Jan 2019 14:17:36 +0000 Subject: [PATCH 33/43] changelog --- changelog.d/4522.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4522.feature diff --git a/changelog.d/4522.feature b/changelog.d/4522.feature new file mode 100644 index 000000000000..daedcd58c442 --- /dev/null +++ b/changelog.d/4522.feature @@ -0,0 +1 @@ +Synapse can now automatically provision TLS certificates via ACME (the protocol used by CAs like Let's Encrypt). From 512dfeb74284f2fc78131608c563052fa2597c52 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 30 Jan 2019 14:45:06 +0000 Subject: [PATCH 34/43] fixes --- synapse/app/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index d96a2c19000c..40c8b2fdbb69 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -25,8 +25,8 @@ from twisted.internet import error, reactor from twisted.protocols.tls import TLSMemoryBIOFactory -from synapse.crypto import context_factory from synapse.app import check_bind_error +from synapse.crypto import context_factory from synapse.util import PreserveLoggingContext from synapse.util.rlimit import change_resource_limit From 5fdaf5cc02a1dfb99d65c94f0100a0abd3f3d506 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 5 Feb 2019 11:33:32 +0000 Subject: [PATCH 35/43] port over --- synapse/app/_base.py | 61 ++++++++++++++++++++++++++++++++ synapse/app/appservice.py | 7 +--- synapse/app/client_reader.py | 13 +------ synapse/app/event_creator.py | 13 +------ synapse/app/federation_reader.py | 13 +------ synapse/app/federation_sender.py | 12 +------ synapse/app/frontend_proxy.py | 13 +------ synapse/app/media_repository.py | 13 +------ synapse/app/pusher.py | 3 +- synapse/app/synchrotron.py | 7 +--- synapse/app/user_dir.py | 13 +------ synapse/config/logger.py | 16 +++------ synapse/server.py | 1 + 13 files changed, 77 insertions(+), 108 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 5b97a54d45a1..40c8b2fdbb69 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -15,19 +15,26 @@ import gc import logging +import signal import sys +import traceback import psutil from daemonize import Daemonize from twisted.internet import error, reactor +from twisted.protocols.tls import TLSMemoryBIOFactory from synapse.app import check_bind_error +from synapse.crypto import context_factory from synapse.util import PreserveLoggingContext from synapse.util.rlimit import change_resource_limit logger = logging.getLogger(__name__) +_sighup_callbacks = [] +register_sighup = _sighup_callbacks.append + def start_worker_reactor(appname, config): """ Run the reactor in the main process @@ -189,3 +196,57 @@ def listen_ssl( logger.info("Synapse now listening on port %d (TLS)", port) return r + + +def refresh_certificate(hs): + """ + Refresh the TLS certificates that Synapse is using by re-reading them from + disk and updating the TLS context factories to use them. + """ + logging.info("Loading certificate from disk...") + hs.config.read_certificate_from_disk() + hs.tls_server_context_factory = context_factory.ServerContextFactory(hs.config) + hs.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( + hs.config + ) + logging.info("Certificate loaded.") + + if hs._listening_services: + logging.info("Updating context factories...") + for i in hs._listening_services: + if isinstance(i.factory, TLSMemoryBIOFactory): + i.factory = TLSMemoryBIOFactory( + hs.tls_server_context_factory, + False, + i.factory.wrappedFactory + ) + logging.info("Context factories updated.") + + +def start(hs, listeners=None): + """ + Start a Synapse server or worker. + """ + try: + # Set up the SIGHUP machinery. + if hasattr(signal, "SIGHUP"): + def handle_sighup(*args, **kwargs): + for i in _sighup_callbacks: + i(hs) + + signal.signal(signal.SIGHUP, handle_sighup) + + register_sighup(refresh_certificate) + + # Load the certificate from disk. + refresh_certificate(hs) + + # It is now safe to start your Synapse. + hs.start_listening(listeners) + hs.get_datastore().start_profiling() + except Exception: + traceback.print_exc(file=sys.stderr) + reactor = hs.get_reactor() + if reactor.running: + reactor.stop() + sys.exit(1) diff --git a/synapse/app/appservice.py b/synapse/app/appservice.py index 8559e141af47..33107f56d137 100644 --- a/synapse/app/appservice.py +++ b/synapse/app/appservice.py @@ -168,12 +168,7 @@ def start(config_options): ) ps.setup() - ps.start_listening(config.worker_listeners) - - def start(): - ps.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ps, config.worker_listeners) _base.start_worker_reactor("synapse-appservice", config) diff --git a/synapse/app/client_reader.py b/synapse/app/client_reader.py index f8a417cb6070..a9d214702286 100644 --- a/synapse/app/client_reader.py +++ b/synapse/app/client_reader.py @@ -25,7 +25,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -173,17 +172,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-client-reader", config) diff --git a/synapse/app/event_creator.py b/synapse/app/event_creator.py index 656e0edc0fbc..b8e51961528b 100644 --- a/synapse/app/event_creator.py +++ b/synapse/app/event_creator.py @@ -25,7 +25,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -194,17 +193,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-event-creator", config) diff --git a/synapse/app/federation_reader.py b/synapse/app/federation_reader.py index 3de27151328c..42886dbfc149 100644 --- a/synapse/app/federation_reader.py +++ b/synapse/app/federation_reader.py @@ -26,7 +26,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.federation.transport.server import TransportLayerServer from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -160,17 +159,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-federation-reader", config) diff --git a/synapse/app/federation_sender.py b/synapse/app/federation_sender.py index d944e0517f2e..a461442fdc41 100644 --- a/synapse/app/federation_sender.py +++ b/synapse/app/federation_sender.py @@ -25,7 +25,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.federation import send_queue from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -192,17 +191,8 @@ def start(config_options): ) ss.setup() + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) _base.start_worker_reactor("synapse-federation-sender", config) diff --git a/synapse/app/frontend_proxy.py b/synapse/app/frontend_proxy.py index d9ef6edc3c6d..d5b954361d9a 100644 --- a/synapse/app/frontend_proxy.py +++ b/synapse/app/frontend_proxy.py @@ -26,7 +26,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseSite @@ -250,17 +249,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-frontend-proxy", config) diff --git a/synapse/app/media_repository.py b/synapse/app/media_repository.py index 4ecf64031bf7..d4cc4e9443c7 100644 --- a/synapse/app/media_repository.py +++ b/synapse/app/media_repository.py @@ -26,7 +26,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy from synapse.metrics.resource import METRICS_PREFIX, MetricsResource @@ -160,17 +159,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-media-repository", config) diff --git a/synapse/app/pusher.py b/synapse/app/pusher.py index 83b0863f00f4..cbf0d67f51fa 100644 --- a/synapse/app/pusher.py +++ b/synapse/app/pusher.py @@ -224,11 +224,10 @@ def start(config_options): ) ps.setup() - ps.start_listening(config.worker_listeners) def start(): + _base.start(ps, config.worker_listeners) ps.get_pusherpool().start() - ps.get_datastore().start_profiling() reactor.callWhenRunning(start) diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index 0354e82bf850..9163b56d86fb 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -445,12 +445,7 @@ def start(config_options): ) ss.setup() - ss.start_listening(config.worker_listeners) - - def start(): - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-synchrotron", config) diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py index 176d55a78333..d1ab9512cd07 100644 --- a/synapse/app/user_dir.py +++ b/synapse/app/user_dir.py @@ -26,7 +26,6 @@ from synapse.config._base import ConfigError from synapse.config.homeserver import HomeServerConfig from synapse.config.logger import setup_logging -from synapse.crypto import context_factory from synapse.http.server import JsonResource from synapse.http.site import SynapseSite from synapse.metrics import RegistryProxy @@ -220,17 +219,7 @@ def start(config_options): ) ss.setup() - - def start(): - ss.config.read_certificate_from_disk() - ss.tls_server_context_factory = context_factory.ServerContextFactory(config) - ss.tls_client_options_factory = context_factory.ClientTLSOptionsFactory( - config - ) - ss.start_listening(config.worker_listeners) - ss.get_datastore().start_profiling() - - reactor.callWhenRunning(start) + reactor.callWhenRunning(_base.start, ss, config.worker_listeners) _base.start_worker_reactor("synapse-user-dir", config) diff --git a/synapse/config/logger.py b/synapse/config/logger.py index a795e39b1acc..4b938053fb75 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -15,7 +15,6 @@ import logging import logging.config import os -import signal import sys from string import Template @@ -24,6 +23,7 @@ from twisted.logger import STDLibLogObserver, globalLogBeginner import synapse +from synapse.app import _base as appbase from synapse.util.logcontext import LoggingContextFilter from synapse.util.versionstring import get_version_string @@ -127,7 +127,7 @@ def generate_files(self, config): ) -def setup_logging(config, use_worker_options=False, register_sighup=None): +def setup_logging(config, use_worker_options=False): """ Set up python logging Args: @@ -140,12 +140,6 @@ def setup_logging(config, use_worker_options=False, register_sighup=None): register_sighup (func | None): Function to call to register a sighup handler. """ - if not register_sighup: - if getattr(signal, "SIGHUP"): - register_sighup = lambda x: signal.signal(signal.SIGHUP, x) - else: - register_sighup = lambda x: None - log_config = (config.worker_log_config if use_worker_options else config.log_config) log_file = (config.worker_log_file if use_worker_options @@ -187,7 +181,7 @@ def sighup(signum, stack): else: handler = logging.StreamHandler() - def sighup(signum, stack): + def sighup(*args): pass handler.setFormatter(formatter) @@ -200,14 +194,14 @@ def load_log_config(): with open(log_config, 'r') as f: logging.config.dictConfig(yaml.load(f)) - def sighup(signum, stack): + def sighup(*args): # it might be better to use a file watcher or something for this. load_log_config() logging.info("Reloaded log config from %s due to SIGHUP", log_config) load_log_config() - register_sighup(sighup) + appbase.register_sighup(sighup) # make sure that the first thing we log is a thing we can grep backwards # for diff --git a/synapse/server.py b/synapse/server.py index 6c521016163f..cb35a7bab3cc 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -196,6 +196,7 @@ def __init__(self, hostname, reactor=None, **kwargs): self._reactor = reactor self.hostname = hostname self._building = {} + self._listening_services = [] self.clock = Clock(reactor) self.distributor = Distributor() From 3fa83ab47f8003f29dd15176ed04116645c4e559 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 5 Feb 2019 11:44:47 +0000 Subject: [PATCH 36/43] changelog --- changelog.d/4567.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4567.misc diff --git a/changelog.d/4567.misc b/changelog.d/4567.misc new file mode 100644 index 000000000000..96a2e0aefc16 --- /dev/null +++ b/changelog.d/4567.misc @@ -0,0 +1 @@ +Reduce duplication of ``synapse.app`` code. From 177969689d82716236d338519a8d7c70feca6ac9 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 5 Feb 2019 11:50:12 +0000 Subject: [PATCH 37/43] fix --- synapse/app/homeserver.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 250a17cef865..33a96ae51b6d 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -17,7 +17,6 @@ import gc import logging import os -import signal import sys import traceback @@ -328,20 +327,11 @@ def setup(config_options): # generating config files and shouldn't try to continue. sys.exit(0) - sighup_callbacks = [] synapse.config.logger.setup_logging( config, - use_worker_options=False, - register_sighup=sighup_callbacks.append + use_worker_options=False ) - def handle_sighup(*args, **kwargs): - for i in sighup_callbacks: - i(*args, **kwargs) - - if hasattr(signal, "SIGHUP"): - signal.signal(signal.SIGHUP, handle_sighup) - events.USE_FROZEN_DICTS = config.use_frozen_dicts database_engine = create_engine(config.database_config) @@ -400,8 +390,6 @@ def refresh_certificate(*args): ) logging.info("Context factories updated.") - sighup_callbacks.append(refresh_certificate) - @defer.inlineCallbacks def start(): try: From 89fba92a15ea81a895d021383d65ae50094c738c Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 8 Feb 2019 14:47:03 +0000 Subject: [PATCH 38/43] some cleanup --- synapse/app/_base.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 40c8b2fdbb69..59ffe4a0af1c 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -33,7 +33,16 @@ logger = logging.getLogger(__name__) _sighup_callbacks = [] -register_sighup = _sighup_callbacks.append + +def register_sighup(func): + """ + Register a function to be called when a SIGHUP occurs. + + Args: + func (function): Function to be called when sent a SIGHUP signal. + Will be called with a single argument, the homeserver. + """ + _sighup_callbacks.append(func) def start_worker_reactor(appname, config): @@ -211,21 +220,13 @@ def refresh_certificate(hs): ) logging.info("Certificate loaded.") - if hs._listening_services: - logging.info("Updating context factories...") - for i in hs._listening_services: - if isinstance(i.factory, TLSMemoryBIOFactory): - i.factory = TLSMemoryBIOFactory( - hs.tls_server_context_factory, - False, - i.factory.wrappedFactory - ) - logging.info("Context factories updated.") - - def start(hs, listeners=None): """ Start a Synapse server or worker. + + Args: + hs (synapse.server.HomeServer) + listeners (list[dict]): Listener configuration ('listeners' in homeserver.yaml) """ try: # Set up the SIGHUP machinery. From b232c178e85ad924dcbd14a89cda9f47a281372d Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 8 Feb 2019 14:48:05 +0000 Subject: [PATCH 39/43] some cleanup --- synapse/app/_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 59ffe4a0af1c..3cbb0030357b 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -23,7 +23,6 @@ from daemonize import Daemonize from twisted.internet import error, reactor -from twisted.protocols.tls import TLSMemoryBIOFactory from synapse.app import check_bind_error from synapse.crypto import context_factory @@ -34,6 +33,7 @@ _sighup_callbacks = [] + def register_sighup(func): """ Register a function to be called when a SIGHUP occurs. @@ -220,6 +220,7 @@ def refresh_certificate(hs): ) logging.info("Certificate loaded.") + def start(hs, listeners=None): """ Start a Synapse server or worker. From ffdac50cc761c3fa7501fb0b977957e18c5f8b73 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 8 Feb 2019 14:54:54 +0000 Subject: [PATCH 40/43] some cleanup --- synapse/server.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/server.py b/synapse/server.py index cb35a7bab3cc..6c521016163f 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -196,7 +196,6 @@ def __init__(self, hostname, reactor=None, **kwargs): self._reactor = reactor self.hostname = hostname self._building = {} - self._listening_services = [] self.clock = Clock(reactor) self.distributor = Distributor() From fcb2fe13a982c837c4abaa76fd1856ee4d896f69 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 8 Feb 2019 18:57:48 +0000 Subject: [PATCH 41/43] Update _base.py --- synapse/app/_base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index dfe2e8f9bd06..3b9ba5e9a3ab 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -232,6 +232,7 @@ def refresh_certificate(hs): ) logging.info("Context factories updated.") + def start(hs, listeners=None): """ Start a Synapse server or worker. From 565b3d224085341faaee011674654b637259948c Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 11 Feb 2019 20:12:21 +1100 Subject: [PATCH 42/43] some docs --- synapse/app/_base.py | 7 +++++++ synapse/server.py | 2 ++ 2 files changed, 9 insertions(+) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 3b9ba5e9a3ab..62c633146fe6 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -224,7 +224,14 @@ def refresh_certificate(hs): if hs._listening_services: logging.info("Updating context factories...") for i in hs._listening_services: + # When you listenSSL, it doesn't make an SSL port but a TCP one with + # a TLS wrapping factory around the factory you actually want to get + # requests. This factory attribute is public but missing from + # Twisted's documentation. if isinstance(i.factory, TLSMemoryBIOFactory): + # We want to replace TLS factories with a new one, with the new + # TLS configuration. We do this by reaching in and pulling out + # the wrappedFactory, and then re-wrapping it. i.factory = TLSMemoryBIOFactory( hs.tls_server_context_factory, False, diff --git a/synapse/server.py b/synapse/server.py index cb35a7bab3cc..a2cf8a91cd1d 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -112,6 +112,8 @@ def build_DEPENDENCY(self) Attributes: config (synapse.config.homeserver.HomeserverConfig): + _listening_services (list[twisted.internet.tcp.Port]): TCP ports that + we are listening on to provide HTTP services. """ __metaclass__ = abc.ABCMeta From 50046cc2c8ec60eab42e84bd4fe3286f52c9a22c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 11 Feb 2019 10:09:04 +0000 Subject: [PATCH 43/43] fix changelog --- changelog.d/4522.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/4522.feature b/changelog.d/4522.feature index daedcd58c442..ef18daf60bd4 100644 --- a/changelog.d/4522.feature +++ b/changelog.d/4522.feature @@ -1 +1 @@ -Synapse can now automatically provision TLS certificates via ACME (the protocol used by CAs like Let's Encrypt). +Synapse's ACME support will now correctly reprovision a certificate that approaches its expiry while Synapse is running.