diff --git a/changelog.d/16985.misc b/changelog.d/16985.misc new file mode 100644 index 00000000000..34d1337a281 --- /dev/null +++ b/changelog.d/16985.misc @@ -0,0 +1 @@ +Allow containers building on top of Synapse's Complement container is use the included PostgreSQL cluster. diff --git a/changelog.d/17002.doc b/changelog.d/17002.doc new file mode 100644 index 00000000000..a1b2ce5e317 --- /dev/null +++ b/changelog.d/17002.doc @@ -0,0 +1 @@ +Remove recommendation for a specific poetry version from contributing guide. diff --git a/changelog.d/17017.misc b/changelog.d/17017.misc new file mode 100644 index 00000000000..c8af23d67a7 --- /dev/null +++ b/changelog.d/17017.misc @@ -0,0 +1 @@ +Patch the db conn pool sooner in tests. diff --git a/docker/complement/conf/postgres.supervisord.conf b/docker/complement/conf/postgres.supervisord.conf index b88bfc772e4..657845dfdbc 100644 --- a/docker/complement/conf/postgres.supervisord.conf +++ b/docker/complement/conf/postgres.supervisord.conf @@ -1,7 +1,7 @@ [program:postgres] command=/usr/local/bin/prefix-log gosu postgres postgres -# Only start if START_POSTGRES=1 +# Only start if START_POSTGRES=true autostart=%(ENV_START_POSTGRES)s # Lower priority number = starts first diff --git a/docker/complement/conf/start_for_complement.sh b/docker/complement/conf/start_for_complement.sh index 7b012ce8abe..cc798a32106 100755 --- a/docker/complement/conf/start_for_complement.sh +++ b/docker/complement/conf/start_for_complement.sh @@ -32,8 +32,9 @@ case "$SYNAPSE_COMPLEMENT_DATABASE" in ;; sqlite|"") - # Configure supervisord not to start Postgres, as we don't need it - export START_POSTGRES=false + # Set START_POSTGRES to false unless it has already been set + # (i.e. by another container image inheriting our own). + export START_POSTGRES=${START_POSTGRES:-false} ;; *) diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md index df6451273aa..ac8a7039d18 100644 --- a/docs/development/contributing_guide.md +++ b/docs/development/contributing_guide.md @@ -68,7 +68,7 @@ Of their installation methods, we recommend ```shell pip install --user pipx -pipx install poetry==1.5.1 # Problems with Poetry 1.6, see https://github.com/matrix-org/synapse/issues/16147 +pipx install poetry ``` but see poetry's [installation instructions](https://python-poetry.org/docs/#installation) diff --git a/docs/postgres.md b/docs/postgres.md index ad7c6a0738d..921bae98775 100644 --- a/docs/postgres.md +++ b/docs/postgres.md @@ -182,7 +182,7 @@ synapse_port_db --sqlite-database homeserver.db.snapshot \ --postgres-config homeserver-postgres.yaml ``` -The flag `--curses` displays a coloured curses progress UI. +The flag `--curses` displays a coloured curses progress UI. (NOTE: if your terminal is too small the script will error out) If the script took a long time to complete, or time has otherwise passed since the original snapshot was taken, repeat the previous steps with a diff --git a/docs/setup/installation.md b/docs/setup/installation.md index 324cdc67b2f..9126874d445 100644 --- a/docs/setup/installation.md +++ b/docs/setup/installation.md @@ -26,7 +26,7 @@ for most users. #### Docker images and Ansible playbooks There is an official synapse image available at - or at [`ghcr.io/element-hq/synapse`](https://ghcr.io/element-hq/synapse) + or at [`ghcr.io/element-hq/synapse`](https://ghcr.io/element-hq/synapse) which can be used with the docker-compose file available at [contrib/docker](https://github.com/element-hq/synapse/tree/develop/contrib/docker). Further information on this including configuration options is available in the README diff --git a/poetry.lock b/poetry.lock index bee0b980578..5332774705b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1575,15 +1575,18 @@ testing-docutils = ["pygments", "pytest (>=7,<8)", "pytest-param-files (>=0.3.4, [[package]] name = "netaddr" -version = "0.9.0" +version = "1.2.1" description = "A network address manipulation library for Python" optional = false -python-versions = "*" +python-versions = ">=3.7" files = [ - {file = "netaddr-0.9.0-py3-none-any.whl", hash = "sha256:5148b1055679d2a1ec070c521b7db82137887fabd6d7e37f5199b44f775c3bb1"}, - {file = "netaddr-0.9.0.tar.gz", hash = "sha256:7b46fa9b1a2d71fd5de9e4a3784ef339700a53a08c8040f08baf5f1194da0128"}, + {file = "netaddr-1.2.1-py3-none-any.whl", hash = "sha256:bd9e9534b0d46af328cf64f0e5a23a5a43fca292df221c85580b27394793496e"}, + {file = "netaddr-1.2.1.tar.gz", hash = "sha256:6eb8fedf0412c6d294d06885c110de945cf4d22d2b510d0404f4e06950857987"}, ] +[package.extras] +nicer-shell = ["ipython"] + [[package]] name = "opentracing" version = "2.4.0" @@ -3070,13 +3073,13 @@ files = [ [[package]] name = "types-jsonschema" -version = "4.21.0.20240118" +version = "4.21.0.20240311" description = "Typing stubs for jsonschema" optional = false python-versions = ">=3.8" files = [ - {file = "types-jsonschema-4.21.0.20240118.tar.gz", hash = "sha256:31aae1b5adc0176c1155c2d4f58348b22d92ae64315e9cc83bd6902168839232"}, - {file = "types_jsonschema-4.21.0.20240118-py3-none-any.whl", hash = "sha256:77a4ac36b0be4f24274d5b9bf0b66208ee771c05f80e34c4641de7d63e8a872d"}, + {file = "types-jsonschema-4.21.0.20240311.tar.gz", hash = "sha256:f7165ce70abd91df490c73b089873afd2899c5e56430ee495b64f851ad01f287"}, + {file = "types_jsonschema-4.21.0.20240311-py3-none-any.whl", hash = "sha256:e872f5661513824edf9698f73a66c9c114713d93eab58699bd0532e7e6db5750"}, ] [package.dependencies] diff --git a/tests/server.py b/tests/server.py index f0cc4206b05..4aaa91e956a 100644 --- a/tests/server.py +++ b/tests/server.py @@ -47,7 +47,7 @@ Union, cast, ) -from unittest.mock import Mock +from unittest.mock import Mock, patch import attr from incremental import Version @@ -55,6 +55,7 @@ from zope.interface import implementer import twisted +from twisted.enterprise import adbapi from twisted.internet import address, tcp, threads, udp from twisted.internet._resolver import SimpleResolverComplexifier from twisted.internet.defer import Deferred, fail, maybeDeferred, succeed @@ -94,8 +95,8 @@ ) from synapse.server import HomeServer from synapse.storage import DataStore -from synapse.storage.database import LoggingDatabaseConnection -from synapse.storage.engines import create_engine +from synapse.storage.database import LoggingDatabaseConnection, make_pool +from synapse.storage.engines import BaseDatabaseEngine, create_engine from synapse.storage.prepare_database import prepare_database from synapse.types import ISynapseReactor, JsonDict from synapse.util import Clock @@ -670,6 +671,53 @@ def validate_connector(connector: tcp.Connector, expected_ip: str) -> None: ) +def make_fake_db_pool( + reactor: ISynapseReactor, + db_config: DatabaseConnectionConfig, + engine: BaseDatabaseEngine, +) -> adbapi.ConnectionPool: + """Wrapper for `make_pool` which builds a pool which runs db queries synchronously. + + For more deterministic testing, we don't use a regular db connection pool: instead + we run all db queries synchronously on the test reactor's main thread. This function + is a drop-in replacement for the normal `make_pool` which builds such a connection + pool. + """ + pool = make_pool(reactor, db_config, engine) + + def runWithConnection( + func: Callable[..., R], *args: Any, **kwargs: Any + ) -> Awaitable[R]: + return threads.deferToThreadPool( + pool._reactor, + pool.threadpool, + pool._runWithConnection, + func, + *args, + **kwargs, + ) + + def runInteraction( + desc: str, func: Callable[..., R], *args: Any, **kwargs: Any + ) -> Awaitable[R]: + return threads.deferToThreadPool( + pool._reactor, + pool.threadpool, + pool._runInteraction, + desc, + func, + *args, + **kwargs, + ) + + pool.runWithConnection = runWithConnection # type: ignore[method-assign] + pool.runInteraction = runInteraction # type: ignore[assignment] + # Replace the thread pool with a threadless 'thread' pool + pool.threadpool = ThreadPool(reactor) + pool.running = True + return pool + + class ThreadPool: """ Threadless thread pool. @@ -706,52 +754,6 @@ def _(res: Any) -> None: return d -def _make_test_homeserver_synchronous(server: HomeServer) -> None: - """ - Make the given test homeserver's database interactions synchronous. - """ - - clock = server.get_clock() - - for database in server.get_datastores().databases: - pool = database._db_pool - - def runWithConnection( - func: Callable[..., R], *args: Any, **kwargs: Any - ) -> Awaitable[R]: - return threads.deferToThreadPool( - pool._reactor, - pool.threadpool, - pool._runWithConnection, - func, - *args, - **kwargs, - ) - - def runInteraction( - desc: str, func: Callable[..., R], *args: Any, **kwargs: Any - ) -> Awaitable[R]: - return threads.deferToThreadPool( - pool._reactor, - pool.threadpool, - pool._runInteraction, - desc, - func, - *args, - **kwargs, - ) - - pool.runWithConnection = runWithConnection # type: ignore[method-assign] - pool.runInteraction = runInteraction # type: ignore[assignment] - # Replace the thread pool with a threadless 'thread' pool - pool.threadpool = ThreadPool(clock._reactor) - pool.running = True - - # We've just changed the Databases to run DB transactions on the same - # thread, so we need to disable the dedicated thread behaviour. - server.get_datastores().main.USE_DEDICATED_DB_THREADS_FOR_EVENT_FETCHING = False - - def get_clock() -> Tuple[ThreadedMemoryReactorClock, Clock]: clock = ThreadedMemoryReactorClock() hs_clock = Clock(clock) @@ -1067,7 +1069,14 @@ def setup_test_homeserver( # Mock TLS hs.tls_server_context_factory = Mock() - hs.setup() + # Patch `make_pool` before initialising the database, to make database transactions + # synchronous for testing. + with patch("synapse.storage.database.make_pool", side_effect=make_fake_db_pool): + hs.setup() + + # Since we've changed the databases to run DB transactions on the same + # thread, we need to stop the event fetcher hogging that one thread. + hs.get_datastores().main.USE_DEDICATED_DB_THREADS_FOR_EVENT_FETCHING = False if USE_POSTGRES_FOR_TESTS: database_pool = hs.get_datastores().databases[0] @@ -1137,9 +1146,6 @@ async def validate_hash(p: str, h: str) -> bool: hs.get_auth_handler().validate_hash = validate_hash # type: ignore[assignment] - # Make the threadpool and database transactions synchronous for testing. - _make_test_homeserver_synchronous(hs) - # Load any configured modules into the homeserver module_api = hs.get_module_api() for module, module_config in hs.config.modules.loaded_modules: