From ecadb7a262e0b43c0317615aaf0d6aef443358a7 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Thu, 13 Jun 2019 23:06:58 +1000 Subject: [PATCH 01/12] Remove from buildkite --- .buildkite/docker-compose.py35.pg94.yaml | 21 --------------------- .buildkite/pipeline.yml | 17 ----------------- 2 files changed, 38 deletions(-) delete mode 100644 .buildkite/docker-compose.py35.pg94.yaml diff --git a/.buildkite/docker-compose.py35.pg94.yaml b/.buildkite/docker-compose.py35.pg94.yaml deleted file mode 100644 index 978aedd1159e..000000000000 --- a/.buildkite/docker-compose.py35.pg94.yaml +++ /dev/null @@ -1,21 +0,0 @@ -version: '3.1' - -services: - - postgres: - image: postgres:9.4 - environment: - POSTGRES_PASSWORD: postgres - - testenv: - image: python:3.5 - depends_on: - - postgres - env_file: .env - environment: - SYNAPSE_POSTGRES_HOST: postgres - SYNAPSE_POSTGRES_USER: postgres - SYNAPSE_POSTGRES_PASSWORD: postgres - working_dir: /app - volumes: - - ..:/app diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8eddf8b93199..e6dd66dddf6b 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -114,23 +114,6 @@ steps: - exit_status: 2 limit: 2 - - label: ":python: 3.5 / :postgres: 9.4" - env: - TRIAL_FLAGS: "-j 4" - command: - - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'" - plugins: - - docker-compose#v2.1.0: - run: testenv - config: - - .buildkite/docker-compose.py35.pg94.yaml - retry: - automatic: - - exit_status: -1 - limit: 2 - - exit_status: 2 - limit: 2 - - label: ":python: 3.5 / :postgres: 9.5" env: TRIAL_FLAGS: "-j 4" From d6b275e3b0355ccfc711dfaae0d71776aa21d96a Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Thu, 13 Jun 2019 23:07:13 +1000 Subject: [PATCH 02/12] update our contributing docs --- CONTRIBUTING.rst | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 9a283ced6eb6..2c44422a0e84 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -30,21 +30,20 @@ use github's pull request workflow to review the contribution, and either ask you to make any refinements needed or merge it and make them ourselves. The changes will then land on master when we next do a release. -We use `CircleCI `_ and `Travis CI -`_ for continuous integration. All -pull requests to synapse get automatically tested by Travis and CircleCI. -If your change breaks the build, this will be shown in GitHub, so please -keep an eye on the pull request for feedback. +We use `CircleCI `_ and `Buildkite +`_ for continuous integration. +Buildkite builds need to be authorised by a maintainer. If your change breaks +the build, this will be shown in GitHub, so please keep an eye on the pull +request for feedback. To run unit tests in a local development environment, you can use: -- ``tox -e py27`` (requires tox to be installed by ``pip install tox``) for - SQLite-backed Synapse on Python 2.7. -- ``tox -e py35`` for SQLite-backed Synapse on Python 3.5. +- ``tox -e py35`` (requires tox to be installed by ``pip install tox``) + for SQLite-backed Synapse on Python 3.5. - ``tox -e py36`` for SQLite-backed Synapse on Python 3.6. -- ``tox -e py27-postgres`` for PostgreSQL-backed Synapse on Python 2.7 +- ``tox -e py36-postgres`` for PostgreSQL-backed Synapse on Python 3.6 (requires a running local PostgreSQL with access to create databases). -- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 2.7 +- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 3.5 (requires Docker). Entirely self-contained, recommended if you don't want to set up PostgreSQL yourself. From 23f51de133d46cd9249745af5328595ff88fd2a2 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Thu, 13 Jun 2019 23:07:27 +1000 Subject: [PATCH 03/12] make pgtests work on py3 --- docker/Dockerfile-pgtests | 4 ++-- docker/run_pg_tests.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile-pgtests b/docker/Dockerfile-pgtests index 7da8eeb9eb0e..3bfee845c658 100644 --- a/docker/Dockerfile-pgtests +++ b/docker/Dockerfile-pgtests @@ -3,10 +3,10 @@ FROM matrixdotorg/sytest:latest # The Sytest image doesn't come with python, so install that -RUN apt-get -qq install -y python python-dev python-pip +RUN apt-get update && apt-get -qq install -y python3 python3-dev python3-pip # We need tox to run the tests in run_pg_tests.sh -RUN pip install tox +RUN python3 -m pip install tox ADD run_pg_tests.sh /pg_tests.sh ENTRYPOINT /pg_tests.sh diff --git a/docker/run_pg_tests.sh b/docker/run_pg_tests.sh index e77424c41a37..d18d1e4c8e87 100755 --- a/docker/run_pg_tests.sh +++ b/docker/run_pg_tests.sh @@ -17,4 +17,4 @@ su -c '/usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' # Run the tests cd /src export TRIAL_FLAGS="-j 4" -tox --workdir=/tmp -e py27-postgres +tox --workdir=/tmp -e py35-postgres From 05985455fc67bf4359513d0130e6e09fe791348f Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Thu, 13 Jun 2019 23:07:49 +1000 Subject: [PATCH 04/12] Bump the dependency we say that works --- docs/postgres.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/postgres.rst b/docs/postgres.rst index e81e10403f57..33f58e3acea3 100644 --- a/docs/postgres.rst +++ b/docs/postgres.rst @@ -1,7 +1,7 @@ Using Postgres -------------- -Postgres version 9.4 or later is known to work. +Postgres version 9.5 or later is known to work. Install postgres client libraries ================================= @@ -16,7 +16,7 @@ a postgres database. * For other pre-built packages, please consult the documentation from the relevant package. -* If you installed synapse `in a virtualenv +* If you installed synapse `in a virtualenv <../INSTALL.md#installing-from-source>`_, you can install the library with:: ~/synapse/env/bin/pip install matrix-synapse[postgres] From 2d68343e6aa63af6e819d837e2bc2b007572ff5d Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Thu, 13 Jun 2019 23:08:18 +1000 Subject: [PATCH 05/12] we can do native upserts, and also throw a wobbly at running on 9.4 --- synapse/storage/engines/postgres.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 1b97ee74e3a7..289b6bc281ad 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -45,6 +45,10 @@ def on_new_connection(self, db_conn): # together. For example, version 8.1.5 will be returned as 80105 self._version = db_conn.server_version + # Are we on a supported PostgreSQL version? + if self._version < 90500: + raise RuntimeError("Synapse requires PostgreSQL 9.5+ or above.") + db_conn.set_isolation_level( self.module.extensions.ISOLATION_LEVEL_REPEATABLE_READ ) @@ -64,9 +68,9 @@ def on_new_connection(self, db_conn): @property def can_native_upsert(self): """ - Can we use native UPSERTs? This requires PostgreSQL 9.5+. + Can we use native UPSERTs? """ - return self._version >= 90500 + return True def is_deadlock(self, error): if isinstance(error, self.module.DatabaseError): From 5529bd158a0e958a66e7d74951293db6a8f95136 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Thu, 13 Jun 2019 23:08:37 +1000 Subject: [PATCH 06/12] remove this 3 year old junk that is 9.4 specific --- synapse/storage/search.py | 78 --------------------------------------- 1 file changed, 78 deletions(-) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index ff49eaae0278..07182bded3ca 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -40,8 +40,6 @@ class SearchStore(BackgroundUpdateStore): EVENT_SEARCH_UPDATE_NAME = "event_search" EVENT_SEARCH_ORDER_UPDATE_NAME = "event_search_order" - EVENT_SEARCH_USE_GIST_POSTGRES_NAME = "event_search_postgres_gist" - EVENT_SEARCH_USE_GIN_POSTGRES_NAME = "event_search_postgres_gin" def __init__(self, db_conn, hs): super(SearchStore, self).__init__(db_conn, hs) @@ -56,17 +54,6 @@ def __init__(self, db_conn, hs): self.EVENT_SEARCH_ORDER_UPDATE_NAME, self._background_reindex_search_order ) - # we used to have a background update to turn the GIN index into a - # GIST one; we no longer do that (obviously) because we actually want - # a GIN index. However, it's possible that some people might still have - # the background update queued, so we register a handler to clear the - # background update. - self.register_noop_background_update(self.EVENT_SEARCH_USE_GIST_POSTGRES_NAME) - - self.register_background_update_handler( - self.EVENT_SEARCH_USE_GIN_POSTGRES_NAME, self._background_reindex_gin_search - ) - @defer.inlineCallbacks def _background_reindex_search(self, progress, batch_size): # we work through the events table from highest stream id to lowest @@ -168,49 +155,6 @@ def reindex_search_txn(txn): defer.returnValue(result) - @defer.inlineCallbacks - def _background_reindex_gin_search(self, progress, batch_size): - """This handles old synapses which used GIST indexes, if any; - converting them back to be GIN as per the actual schema. - """ - - def create_index(conn): - conn.rollback() - - # we have to set autocommit, because postgres refuses to - # CREATE INDEX CONCURRENTLY without it. - conn.set_session(autocommit=True) - - try: - c = conn.cursor() - - # if we skipped the conversion to GIST, we may already/still - # have an event_search_fts_idx; unfortunately postgres 9.4 - # doesn't support CREATE INDEX IF EXISTS so we just catch the - # exception and ignore it. - import psycopg2 - - try: - c.execute( - "CREATE INDEX CONCURRENTLY event_search_fts_idx" - " ON event_search USING GIN (vector)" - ) - except psycopg2.ProgrammingError as e: - logger.warn( - "Ignoring error %r when trying to switch from GIST to GIN", e - ) - - # we should now be able to delete the GIST index. - c.execute("DROP INDEX IF EXISTS event_search_fts_idx_gist") - finally: - conn.set_session(autocommit=False) - - if isinstance(self.database_engine, PostgresEngine): - yield self.runWithConnection(create_index) - - yield self._end_background_update(self.EVENT_SEARCH_USE_GIN_POSTGRES_NAME) - defer.returnValue(1) - @defer.inlineCallbacks def _background_reindex_search_order(self, progress, batch_size): target_min_stream_id = progress["target_min_stream_id_inclusive"] @@ -341,29 +285,7 @@ def store_search_entries_txn(self, txn, entries): for entry in entries ) - # inserts to a GIN index are normally batched up into a pending - # list, and then all committed together once the list gets to a - # certain size. The trouble with that is that postgres (pre-9.5) - # uses work_mem to determine the length of the list, and work_mem - # is typically very large. - # - # We therefore reduce work_mem while we do the insert. - # - # (postgres 9.5 uses the separate gin_pending_list_limit setting, - # so doesn't suffer the same problem, but changing work_mem will - # be harmless) - # - # Note that we don't need to worry about restoring it on - # exception, because exceptions will cause the transaction to be - # rolled back, including the effects of the SET command. - # - # Also: we use SET rather than SET LOCAL because there's lots of - # other stuff going on in this transaction, which want to have the - # normal work_mem setting. - - txn.execute("SET work_mem='256kB'") txn.executemany(sql, args) - txn.execute("RESET work_mem") elif isinstance(self.database_engine, Sqlite3Engine): sql = ( From 55346522828a02227c3780ccd28fe656356bc949 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Thu, 13 Jun 2019 23:09:52 +1000 Subject: [PATCH 07/12] changelog --- changelog.d/5448.removal | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/5448.removal diff --git a/changelog.d/5448.removal b/changelog.d/5448.removal new file mode 100644 index 000000000000..33b9859daedc --- /dev/null +++ b/changelog.d/5448.removal @@ -0,0 +1 @@ +PostgreSQL 9.4 is no longer supported. Synapse requires Postgres 9.5+ or above for Postgres support. From 761a05464bc11ca43c130229d73f38de58da39ed Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Fri, 14 Jun 2019 01:09:07 +1000 Subject: [PATCH 08/12] upgrade --- UPGRADE.rst | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/UPGRADE.rst b/UPGRADE.rst index 6032a505c945..f820a46f55cd 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -49,6 +49,30 @@ returned by the Client-Server API: # configured on port 443. curl -kv https:///_matrix/client/versions 2>&1 | grep "Server:" +Upgrading to v1.1 +================= + +Minimum Python Version +---------------------- + +Synapse v1.1 has a minimum Python requirement of Python 3.5. Python 3.6 or +Python 3.7 are recommended as they have improved internal string handling, +significantly reducing memory usage. + +If you use current versions of the Matrix.org-distributed Debian packages or +Docker images, action is not required. + +If you install Synapse in a Python virtual environment, please see "Upgrading to +v0.34.0" for notes on setting up a new virtualenv under Python 3. + +Minimum PostgreSQL Version +-------------------------- + +If using PostgreSQL under Synapse, you will need to use PostgreSQL 9.5 or above. +Please see the +`PostgreSQL documentation ` +for more details on upgrading your database. + Upgrading to v1.0 ================= @@ -71,11 +95,11 @@ server in a closed federation. This can be done in one of two ways:- * Configure a whitelist of server domains to trust via ``federation_certificate_verification_whitelist``. See the `sample configuration file `_ -for more details on these settings. +for more details on these settings. Email ----- -When a user requests a password reset, Synapse will send an email to the +When a user requests a password reset, Synapse will send an email to the user to confirm the request. Previous versions of Synapse delegated the job of sending this email to an From 3040c9bf733de9c65b37728a4596a3d7bd7951d7 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Fri, 14 Jun 2019 19:50:10 +1000 Subject: [PATCH 09/12] add deprecation notice --- UPGRADE.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/UPGRADE.rst b/UPGRADE.rst index f820a46f55cd..bebf98b7441b 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -52,6 +52,9 @@ returned by the Client-Server API: Upgrading to v1.1 ================= +Synapse 1.1 removes support for older Python and PostgreSQL versions, as +outlined in `our deprecation notice `. + Minimum Python Version ---------------------- From 211b4ec96d6ba34dec269e796dc4238654ce62f5 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Fri, 14 Jun 2019 19:51:48 +1000 Subject: [PATCH 10/12] fix links --- UPGRADE.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/UPGRADE.rst b/UPGRADE.rst index bebf98b7441b..1fb109a21837 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -53,7 +53,7 @@ Upgrading to v1.1 ================= Synapse 1.1 removes support for older Python and PostgreSQL versions, as -outlined in `our deprecation notice `. +outlined in `our deprecation notice `_. Minimum Python Version ---------------------- @@ -73,7 +73,7 @@ Minimum PostgreSQL Version If using PostgreSQL under Synapse, you will need to use PostgreSQL 9.5 or above. Please see the -`PostgreSQL documentation ` +`PostgreSQL documentation `_ for more details on upgrading your database. Upgrading to v1.0 From 2a52fb2961e3b69a3e3e38c5b5cb3ba69fb17b6d Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Tue, 18 Jun 2019 00:17:10 +1000 Subject: [PATCH 11/12] revert --- synapse/storage/search.py | 78 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index 07182bded3ca..ff49eaae0278 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -40,6 +40,8 @@ class SearchStore(BackgroundUpdateStore): EVENT_SEARCH_UPDATE_NAME = "event_search" EVENT_SEARCH_ORDER_UPDATE_NAME = "event_search_order" + EVENT_SEARCH_USE_GIST_POSTGRES_NAME = "event_search_postgres_gist" + EVENT_SEARCH_USE_GIN_POSTGRES_NAME = "event_search_postgres_gin" def __init__(self, db_conn, hs): super(SearchStore, self).__init__(db_conn, hs) @@ -54,6 +56,17 @@ def __init__(self, db_conn, hs): self.EVENT_SEARCH_ORDER_UPDATE_NAME, self._background_reindex_search_order ) + # we used to have a background update to turn the GIN index into a + # GIST one; we no longer do that (obviously) because we actually want + # a GIN index. However, it's possible that some people might still have + # the background update queued, so we register a handler to clear the + # background update. + self.register_noop_background_update(self.EVENT_SEARCH_USE_GIST_POSTGRES_NAME) + + self.register_background_update_handler( + self.EVENT_SEARCH_USE_GIN_POSTGRES_NAME, self._background_reindex_gin_search + ) + @defer.inlineCallbacks def _background_reindex_search(self, progress, batch_size): # we work through the events table from highest stream id to lowest @@ -155,6 +168,49 @@ def reindex_search_txn(txn): defer.returnValue(result) + @defer.inlineCallbacks + def _background_reindex_gin_search(self, progress, batch_size): + """This handles old synapses which used GIST indexes, if any; + converting them back to be GIN as per the actual schema. + """ + + def create_index(conn): + conn.rollback() + + # we have to set autocommit, because postgres refuses to + # CREATE INDEX CONCURRENTLY without it. + conn.set_session(autocommit=True) + + try: + c = conn.cursor() + + # if we skipped the conversion to GIST, we may already/still + # have an event_search_fts_idx; unfortunately postgres 9.4 + # doesn't support CREATE INDEX IF EXISTS so we just catch the + # exception and ignore it. + import psycopg2 + + try: + c.execute( + "CREATE INDEX CONCURRENTLY event_search_fts_idx" + " ON event_search USING GIN (vector)" + ) + except psycopg2.ProgrammingError as e: + logger.warn( + "Ignoring error %r when trying to switch from GIST to GIN", e + ) + + # we should now be able to delete the GIST index. + c.execute("DROP INDEX IF EXISTS event_search_fts_idx_gist") + finally: + conn.set_session(autocommit=False) + + if isinstance(self.database_engine, PostgresEngine): + yield self.runWithConnection(create_index) + + yield self._end_background_update(self.EVENT_SEARCH_USE_GIN_POSTGRES_NAME) + defer.returnValue(1) + @defer.inlineCallbacks def _background_reindex_search_order(self, progress, batch_size): target_min_stream_id = progress["target_min_stream_id_inclusive"] @@ -285,7 +341,29 @@ def store_search_entries_txn(self, txn, entries): for entry in entries ) + # inserts to a GIN index are normally batched up into a pending + # list, and then all committed together once the list gets to a + # certain size. The trouble with that is that postgres (pre-9.5) + # uses work_mem to determine the length of the list, and work_mem + # is typically very large. + # + # We therefore reduce work_mem while we do the insert. + # + # (postgres 9.5 uses the separate gin_pending_list_limit setting, + # so doesn't suffer the same problem, but changing work_mem will + # be harmless) + # + # Note that we don't need to worry about restoring it on + # exception, because exceptions will cause the transaction to be + # rolled back, including the effects of the SET command. + # + # Also: we use SET rather than SET LOCAL because there's lots of + # other stuff going on in this transaction, which want to have the + # normal work_mem setting. + + txn.execute("SET work_mem='256kB'") txn.executemany(sql, args) + txn.execute("RESET work_mem") elif isinstance(self.database_engine, Sqlite3Engine): sql = ( From cda2b7b588600411d40d776b4695859b1e61f314 Mon Sep 17 00:00:00 2001 From: "Amber H. Brown" Date: Tue, 18 Jun 2019 00:17:25 +1000 Subject: [PATCH 12/12] mod --- synapse/storage/search.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/synapse/storage/search.py b/synapse/storage/search.py index ff49eaae0278..10a27c207a2f 100644 --- a/synapse/storage/search.py +++ b/synapse/storage/search.py @@ -341,29 +341,7 @@ def store_search_entries_txn(self, txn, entries): for entry in entries ) - # inserts to a GIN index are normally batched up into a pending - # list, and then all committed together once the list gets to a - # certain size. The trouble with that is that postgres (pre-9.5) - # uses work_mem to determine the length of the list, and work_mem - # is typically very large. - # - # We therefore reduce work_mem while we do the insert. - # - # (postgres 9.5 uses the separate gin_pending_list_limit setting, - # so doesn't suffer the same problem, but changing work_mem will - # be harmless) - # - # Note that we don't need to worry about restoring it on - # exception, because exceptions will cause the transaction to be - # rolled back, including the effects of the SET command. - # - # Also: we use SET rather than SET LOCAL because there's lots of - # other stuff going on in this transaction, which want to have the - # normal work_mem setting. - - txn.execute("SET work_mem='256kB'") txn.executemany(sql, args) - txn.execute("RESET work_mem") elif isinstance(self.database_engine, Sqlite3Engine): sql = (