From 03f2f75164cff28a05b041ab20a61669b9ace41b Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 18 Dec 2018 13:40:26 +1100 Subject: [PATCH 01/40] add pg9.5+ upserts --- synapse/storage/_base.py | 28 ++++++++++++++++++++++++++++ synapse/storage/engines/postgres.py | 13 +++++++++++++ 2 files changed, 41 insertions(+) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 1d3069b14350..80e8bfb7885a 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -519,6 +519,17 @@ def _simple_upsert(self, table, keyvalues, values, Deferred(bool): True if a new entry was created, False if an existing one was updated. """ + # On PostgreSQL 9.5+ we can do UPSERTs + if isinstance(self.database_engine, PostgresEngine): + if self.database_engine._server_version >= (9, 5, 0): + # We don't put this in a loop as it is guaranteed to be atomic, + # so if we get an IntegrityError, it's unrelated. + result = yield self.runInteraction( + desc, + self._simple_upsert_txn_native_upsert, table, keyvalues, values, insertion_values + ) + defer.returnValue(result) + attempts = 0 while True: try: @@ -575,6 +586,23 @@ def _simple_upsert_txn(self, txn, table, keyvalues, values, insertion_values={}, # successfully inserted return True + def _simple_upsert_txn_native_upsert(self, txn, table, keyvalues, values, insertion_values={}): + + allvalues = {} + allvalues.update(keyvalues) + allvalues.update(values) + allvalues.update(insertion_values) + + sql = "INSERT INTO %s (%s) VALUES (%s) ON CONFLICT (%s) DO UPDATE SET %s" % ( + table, + ", ".join(k for k in allvalues), + ", ".join("?" for _ in allvalues), + ", ".join(k for k in keyvalues), + ", ".join(k + "=EXCLUDED." + k for k in values) + ) + txn.execute(sql, list(allvalues.values())) + return True + def _simple_select_one(self, table, keyvalues, retcols, allow_none=False, desc="_simple_select_one"): """Executes a SELECT query on the named table, which is expected to diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 42225f8a2a7e..8c8a8f280683 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -34,10 +34,23 @@ def check_database(self, txn): % (rows[0][0],) ) + def convert_param_style(self, sql): return sql.replace("?", "%s") def on_new_connection(self, db_conn): + + # Get the version of PostgreSQL that we're using. As per the psycopg2 + # docs: The number is formed by converting the major, minor, and + # revision numbers into two-decimal-digit numbers and appending them + # together. For example, version 8.1.5 will be returned as 80105. + server_version = str(db_conn.server_version) + self._server_version = ( + int(server_version[:-4], 10), + int(server_version[-4:-2], 10), + int(server_version[-2:], 10) + ) + db_conn.set_isolation_level( self.module.extensions.ISOLATION_LEVEL_REPEATABLE_READ ) From b0c720e93026310ccdea32ea0c10fb4ff105fe78 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 18 Dec 2018 23:24:11 +1100 Subject: [PATCH 02/40] add sqlite 3.24+ upserts --- synapse/storage/_base.py | 33 +++++++++++++++++++---------- synapse/storage/engines/postgres.py | 2 +- synapse/storage/engines/sqlite3.py | 4 ++++ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 80e8bfb7885a..d314c6d3e5d7 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -26,7 +26,7 @@ from twisted.internet import defer from synapse.api.errors import StoreError -from synapse.storage.engines import PostgresEngine +from synapse.storage.engines import PostgresEngine, Sqlite3Engine from synapse.util.caches.descriptors import Cache from synapse.util.logcontext import LoggingContext, PreserveLoggingContext from synapse.util.stringutils import exception_to_unicode @@ -519,16 +519,25 @@ def _simple_upsert(self, table, keyvalues, values, Deferred(bool): True if a new entry was created, False if an existing one was updated. """ - # On PostgreSQL 9.5+ we can do UPSERTs + # Can we perform a native (unlocked) UPSERT? + native_upsert = False + if isinstance(self.database_engine, PostgresEngine): - if self.database_engine._server_version >= (9, 5, 0): - # We don't put this in a loop as it is guaranteed to be atomic, - # so if we get an IntegrityError, it's unrelated. - result = yield self.runInteraction( - desc, - self._simple_upsert_txn_native_upsert, table, keyvalues, values, insertion_values - ) - defer.returnValue(result) + # On PostgreSQL 9.5+ we can do native UPSERTs + native_upsert = self.database_engine._version >= (9, 5, 0) + + if isinstance(self.database_engine, Sqlite3Engine): + # On SQLite 3.24+ we can do native UPSERTs + native_upsert = self.database_engine._version >= (3, 24, 0) + + if native_upsert: + # We don't put this in a loop as it is guaranteed to be atomic, so + # if we get an IntegrityError, it's unrelated. + result = yield self.runInteraction( + desc, + self._simple_upsert_txn_native_upsert, table, keyvalues, values, insertion_values + ) + defer.returnValue(result) attempts = 0 while True: @@ -587,7 +596,9 @@ def _simple_upsert_txn(self, txn, table, keyvalues, values, insertion_values={}, return True def _simple_upsert_txn_native_upsert(self, txn, table, keyvalues, values, insertion_values={}): - + """ + Use the native UPSERT functionality in recent PostgreSQL and SQLite versions. + """ allvalues = {} allvalues.update(keyvalues) allvalues.update(values) diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 8c8a8f280683..78199abb9468 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -45,7 +45,7 @@ def on_new_connection(self, db_conn): # revision numbers into two-decimal-digit numbers and appending them # together. For example, version 8.1.5 will be returned as 80105. server_version = str(db_conn.server_version) - self._server_version = ( + self._version = ( int(server_version[:-4], 10), int(server_version[-4:-2], 10), int(server_version[-2:], 10) diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 19949fc4743d..95dcf4b9a549 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -30,6 +30,10 @@ def __init__(self, database_module, database_config): self._current_state_group_id = None self._current_state_group_id_lock = threading.Lock() + # Get the version of SQLite we're using. + import sqlite3 + self._version = sqlite3.sqlite_version_info + def check_database(self, txn): pass From 092034a44d61d47f92dc825373eaa9e74f639010 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 18 Dec 2018 23:25:06 +1100 Subject: [PATCH 03/40] changelog --- changelog.d/4306.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4306.misc diff --git a/changelog.d/4306.misc b/changelog.d/4306.misc new file mode 100644 index 000000000000..58130b61908b --- /dev/null +++ b/changelog.d/4306.misc @@ -0,0 +1 @@ +Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+ and SQLite 3.24+. From 82bd14e739851e8f2c82f4d044756ffb17e25214 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 18 Dec 2018 23:33:32 +1100 Subject: [PATCH 04/40] refactor feature checking into the engine --- synapse/storage/_base.py | 13 +------------ synapse/storage/engines/postgres.py | 7 +++++++ synapse/storage/engines/sqlite3.py | 10 +++++++--- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index d314c6d3e5d7..92cce712ddc9 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -519,18 +519,7 @@ def _simple_upsert(self, table, keyvalues, values, Deferred(bool): True if a new entry was created, False if an existing one was updated. """ - # Can we perform a native (unlocked) UPSERT? - native_upsert = False - - if isinstance(self.database_engine, PostgresEngine): - # On PostgreSQL 9.5+ we can do native UPSERTs - native_upsert = self.database_engine._version >= (9, 5, 0) - - if isinstance(self.database_engine, Sqlite3Engine): - # On SQLite 3.24+ we can do native UPSERTs - native_upsert = self.database_engine._version >= (3, 24, 0) - - if native_upsert: + if self.database_engine.can_native_upsert: # We don't put this in a loop as it is guaranteed to be atomic, so # if we get an IntegrityError, it's unrelated. result = yield self.runInteraction( diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 78199abb9468..ebb4c61a5d86 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -67,6 +67,13 @@ def on_new_connection(self, db_conn): cursor.close() + @property + def can_native_upsert(self): + """ + Can we use native UPSERTs? This requires PostgreSQL 9.5+. + """ + return self._version >= (9, 5, 0) + def is_deadlock(self, error): if isinstance(error, self.module.DatabaseError): # https://www.postgresql.org/docs/current/static/errcodes-appendix.html diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index 95dcf4b9a549..a4d16313902d 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -30,9 +30,13 @@ def __init__(self, database_module, database_config): self._current_state_group_id = None self._current_state_group_id_lock = threading.Lock() - # Get the version of SQLite we're using. - import sqlite3 - self._version = sqlite3.sqlite_version_info + @property + def can_native_upsert(self): + """ + Do we support native UPSERTs? This requires SQLite3 3.24+. + """ + from sqlite3 import sqlite_version_info + return sqlite_version_info >= (3, 24, 0) def check_database(self, txn): pass From 43eef603cff66918dc04432e9c454a0140610662 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 18 Dec 2018 23:38:13 +1100 Subject: [PATCH 05/40] fixup pep8 --- synapse/storage/engines/postgres.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index ebb4c61a5d86..e941608297ff 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -34,7 +34,6 @@ def check_database(self, txn): % (rows[0][0],) ) - def convert_param_style(self, sql): return sql.replace("?", "%s") From e1dd7fea4c892e01b2b06aebc108d22384816668 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 18 Dec 2018 23:42:00 +1100 Subject: [PATCH 06/40] fixup sqlite3 --- synapse/storage/engines/sqlite3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index a4d16313902d..67f070806f20 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -35,8 +35,8 @@ def can_native_upsert(self): """ Do we support native UPSERTs? This requires SQLite3 3.24+. """ - from sqlite3 import sqlite_version_info - return sqlite_version_info >= (3, 24, 0) + from sqlite3 import dbapi2 + return dbapi2.sqlite_version_info >= (3, 24, 0) def check_database(self, txn): pass From 5c5061eb443434e00095e2b2480429a0e48f3ef3 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 18 Dec 2018 23:53:44 +1100 Subject: [PATCH 07/40] don't shadow a global import name --- synapse/storage/engines/__init__.py | 2 +- synapse/storage/engines/{sqlite3.py => sqlite.py} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename synapse/storage/engines/{sqlite3.py => sqlite.py} (96%) diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index e2f9de8451eb..ff5ef97ca86d 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -18,7 +18,7 @@ from ._base import IncorrectDatabaseSetup from .postgres import PostgresEngine -from .sqlite3 import Sqlite3Engine +from .sqlite import Sqlite3Engine SUPPORTED_MODULE = { "sqlite3": Sqlite3Engine, diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite.py similarity index 96% rename from synapse/storage/engines/sqlite3.py rename to synapse/storage/engines/sqlite.py index 67f070806f20..a4d16313902d 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite.py @@ -35,8 +35,8 @@ def can_native_upsert(self): """ Do we support native UPSERTs? This requires SQLite3 3.24+. """ - from sqlite3 import dbapi2 - return dbapi2.sqlite_version_info >= (3, 24, 0) + from sqlite3 import sqlite_version_info + return sqlite_version_info >= (3, 24, 0) def check_database(self, txn): pass From 51983524014c3d12c622ed6ab50819966ac69b90 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 18 Dec 2018 23:54:29 +1100 Subject: [PATCH 08/40] fix up gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3b2252ad8a5a..c7ea28dc2e96 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,8 @@ homeserver*.yaml *.tls.dh *.tls.key -.coverage +.coverage* +!.coveragerc htmlcov demo/*/*.db From 6e2e7ff235818800b10af138ba374f5af7e927ce Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 19 Dec 2018 04:33:10 +1100 Subject: [PATCH 09/40] black --- synapse/storage/_base.py | 45 +++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 92cce712ddc9..2fca119fefa8 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -26,7 +26,7 @@ from twisted.internet import defer from synapse.api.errors import StoreError -from synapse.storage.engines import PostgresEngine, Sqlite3Engine +from synapse.storage.engines import PostgresEngine from synapse.util.caches.descriptors import Cache from synapse.util.logcontext import LoggingContext, PreserveLoggingContext from synapse.util.stringutils import exception_to_unicode @@ -494,8 +494,15 @@ def _simple_insert_many_txn(txn, table, values): txn.executemany(sql, vals) @defer.inlineCallbacks - def _simple_upsert(self, table, keyvalues, values, - insertion_values={}, desc="_simple_upsert", lock=True): + def _simple_upsert( + self, + table, + keyvalues, + values, + insertion_values={}, + desc="_simple_upsert", + lock=True, + ): """ `lock` should generally be set to True (the default), but can be set @@ -524,7 +531,11 @@ def _simple_upsert(self, table, keyvalues, values, # if we get an IntegrityError, it's unrelated. result = yield self.runInteraction( desc, - self._simple_upsert_txn_native_upsert, table, keyvalues, values, insertion_values + self._simple_upsert_txn_native_upsert, + table, + keyvalues, + values, + insertion_values, ) defer.returnValue(result) @@ -533,8 +544,12 @@ def _simple_upsert(self, table, keyvalues, values, try: result = yield self.runInteraction( desc, - self._simple_upsert_txn, table, keyvalues, values, insertion_values, - lock=lock + self._simple_upsert_txn, + table, + keyvalues, + values, + insertion_values, + lock=lock, ) defer.returnValue(result) except self.database_engine.module.IntegrityError as e: @@ -546,12 +561,12 @@ def _simple_upsert(self, table, keyvalues, values, # presumably we raced with another transaction: let's retry. logger.warn( - "IntegrityError when upserting into %s; retrying: %s", - table, e + "IntegrityError when upserting into %s; retrying: %s", table, e ) - def _simple_upsert_txn(self, txn, table, keyvalues, values, insertion_values={}, - lock=True): + def _simple_upsert_txn( + self, txn, table, keyvalues, values, insertion_values={}, lock=True + ): # We need to lock the table :(, unless we're *really* careful if lock: self.database_engine.lock_table(txn, table) @@ -560,7 +575,7 @@ def _simple_upsert_txn(self, txn, table, keyvalues, values, insertion_values={}, sql = "UPDATE %s SET %s WHERE %s" % ( table, ", ".join("%s = ?" % (k,) for k in values), - " AND ".join("%s = ?" % (k,) for k in keyvalues) + " AND ".join("%s = ?" % (k,) for k in keyvalues), ) sqlargs = list(values.values()) + list(keyvalues.values()) @@ -578,13 +593,15 @@ def _simple_upsert_txn(self, txn, table, keyvalues, values, insertion_values={}, sql = "INSERT INTO %s (%s) VALUES (%s)" % ( table, ", ".join(k for k in allvalues), - ", ".join("?" for _ in allvalues) + ", ".join("?" for _ in allvalues), ) txn.execute(sql, list(allvalues.values())) # successfully inserted return True - def _simple_upsert_txn_native_upsert(self, txn, table, keyvalues, values, insertion_values={}): + def _simple_upsert_txn_native_upsert( + self, txn, table, keyvalues, values, insertion_values={} + ): """ Use the native UPSERT functionality in recent PostgreSQL and SQLite versions. """ @@ -598,7 +615,7 @@ def _simple_upsert_txn_native_upsert(self, txn, table, keyvalues, values, insert ", ".join(k for k in allvalues), ", ".join("?" for _ in allvalues), ", ".join(k for k in keyvalues), - ", ".join(k + "=EXCLUDED." + k for k in values) + ", ".join(k + "=EXCLUDED." + k for k in values), ) txn.execute(sql, list(allvalues.values())) return True From 3ac5bcc8c815dfcd03ddc0fd2d2cc07d587109a3 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 19 Dec 2018 05:08:50 +1100 Subject: [PATCH 10/40] try this? --- .coveragerc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.coveragerc b/.coveragerc index ca333961f3b4..795f1f5f43df 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,10 +3,6 @@ branch = True parallel = True source = synapse -[paths] -source= - coverage - [report] precision = 2 ignore_errors = True From 98b00e6909b1cfde78778e7b2f517f82aad856c1 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 19 Dec 2018 05:46:59 +1100 Subject: [PATCH 11/40] try this? --- .gitignore | 1 + tox.ini | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index c7ea28dc2e96..929d61a36858 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ homeserver*.yaml *.tls.key .coverage* +coverage.* !.coveragerc htmlcov diff --git a/tox.ini b/tox.ini index 3c398dc3bafc..c85910c4a0ba 100644 --- a/tox.ini +++ b/tox.ini @@ -150,4 +150,6 @@ deps = codecov commands = coverage combine + coverage xml + sed -i -e 's/name="."/name="synapse"/' coverage.xml codecov -X gcov From 1e0008b3036d1db68c47a3d08435a018142fa612 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 19 Dec 2018 06:05:10 +1100 Subject: [PATCH 12/40] try this? --- .coveragerc | 4 +++- tox.ini | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.coveragerc b/.coveragerc index 795f1f5f43df..be41d079c406 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,8 +1,10 @@ [run] branch = True parallel = True -source = synapse +source = . +include = synapse/* [report] precision = 2 ignore_errors = True +include = synapse/* diff --git a/tox.ini b/tox.ini index c85910c4a0ba..bad4155d6f79 100644 --- a/tox.ini +++ b/tox.ini @@ -151,5 +151,4 @@ deps = commands = coverage combine coverage xml - sed -i -e 's/name="."/name="synapse"/' coverage.xml codecov -X gcov From faae2e12ee12453754147a994de8aeb116f314a9 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 21 Dec 2018 16:44:37 +1100 Subject: [PATCH 13/40] test --- synapse/storage/_base.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 2fca119fefa8..edca82be00f7 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -502,6 +502,7 @@ def _simple_upsert( insertion_values={}, desc="_simple_upsert", lock=True, + best_effort=False, ): """ @@ -522,6 +523,7 @@ def _simple_upsert( insertion_values (dict): additional key/values to use only when inserting lock (bool): True to lock the table when doing the upsert. + best_effort (bool): If we run into a transaction error, do we stop trying? Returns: Deferred(bool): True if a new entry was created, False if an existing one was updated. @@ -529,15 +531,19 @@ def _simple_upsert( if self.database_engine.can_native_upsert: # We don't put this in a loop as it is guaranteed to be atomic, so # if we get an IntegrityError, it's unrelated. - result = yield self.runInteraction( - desc, - self._simple_upsert_txn_native_upsert, - table, - keyvalues, - values, - insertion_values, - ) - defer.returnValue(result) + try: + result = yield self.runInteraction( + desc, + self._simple_upsert_txn_native_upsert, + table, + keyvalues, + values, + insertion_values, + ) + defer.returnValue(result) + except Exception as e: + logger.warn("Got a %s, %s", (repr(e), str(e.args))) + raise attempts = 0 while True: From fd1257064fc9515a3ac10e7aa495683a7fb94efd Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 21 Dec 2018 17:50:09 +1100 Subject: [PATCH 14/40] fix --- synapse/storage/_base.py | 49 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index edca82be00f7..edb1613a569f 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -558,7 +558,31 @@ def _simple_upsert( lock=lock, ) defer.returnValue(result) - except self.database_engine.module.IntegrityError as e: + except ( + self.database_engine.module.IntegrityError, + self.database_engine.module.OperationalError, + ) as e: + + if isinstance(e, self.database_engine.module.OperationalError): + # We only care about serialization errors, so check for it + if ( + e.args[0] + == "could not serialize access due to concurrent update" + ): + # A concurrent update problem is when we try and do a + # native UPSERT but the row has changed from under us. + # We can either retry, or give up if asked to do so. + if best_effort: + # If it's a concurrent-update problem, and this is + # marked as 'best effort' (i.e. if there's a race, + # then the one we raced with will suffice), then + # pretend that we succeeded. + defer.returnValue(False) + else: + # Otherwise, raise, because it's a real OperationalError + # and we will need to be rolled back and retried. + raise + attempts += 1 if attempts >= 5: # don't retry forever, because things other than races @@ -567,11 +591,32 @@ def _simple_upsert( # presumably we raced with another transaction: let's retry. logger.warn( - "IntegrityError when upserting into %s; retrying: %s", table, e + "%s when upserting into %s; retrying: %s", e.__name__, table, e ) def _simple_upsert_txn( self, txn, table, keyvalues, values, insertion_values={}, lock=True + ): + """ + Pick the UPSERT method which works best on the platform. Either the + native one (Pg9.5+, recent SQLites), or fall back to an emulated method. + """ + if self.database_engine.can_native_upsert: + return self._simple_upsert_txn_native_upsert( + txn, table, keyvalues, values, insertion_values=insertion_values + ) + else: + return self._simple_upsert_txn_emulated( + txn, + table, + keyvalues, + values, + insertion_values=insertion_values, + lock=lock, + ) + + def _simple_upsert_txn_emulated( + self, txn, table, keyvalues, values, insertion_values={}, lock=True ): # We need to lock the table :(, unless we're *really* careful if lock: From 7200d0ae06b340243ded78ce05f9663a29392b06 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 22 Dec 2018 00:21:45 +1100 Subject: [PATCH 15/40] fix for mau? --- synapse/storage/monthly_active_users.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py index d6fc8edd4c36..cff3ad0f3094 100644 --- a/synapse/storage/monthly_active_users.py +++ b/synapse/storage/monthly_active_users.py @@ -237,7 +237,9 @@ def upsert_monthly_active_user_txn(self, txn, user_id): # Am consciously deciding to lock the table on the basis that is ought # never be a big table and alternative approaches (batching multiple # upserts into a single txn) introduced a lot of extra complexity. - # See https://github.com/matrix-org/synapse/issues/3854 for more + # See https://github.com/matrix-org/synapse/issues/3854 for more. + # If we support native upserts, we'll not lock, but also not retry + # on any races, by setting best_effort=True. is_insert = self._simple_upsert_txn( txn, table="monthly_active_users", @@ -247,6 +249,7 @@ def upsert_monthly_active_user_txn(self, txn, user_id): values={ "timestamp": int(self._clock.time_msec()), }, + best_effort=True, ) return is_insert From 76bc34aca171399271aaa02a69f59d16420e48da Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 22 Dec 2018 02:11:30 +1100 Subject: [PATCH 16/40] small fix --- .coveragerc | 1 - 1 file changed, 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index c0cc3d919815..e9460a340a65 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,7 +1,6 @@ [run] branch = True parallel = True -source = . include = synapse/* [report] From f3a0ab87c3a0b3fc501b831daf641537efb5ce7d Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 27 Dec 2018 22:57:53 +1100 Subject: [PATCH 17/40] fix the client ips batch --- synapse/storage/_base.py | 10 +++++++--- synapse/storage/client_ips.py | 9 +++++++++ synapse/storage/engines/sqlite.py | 13 ++++++++++--- synapse/storage/prepare_database.py | 2 +- .../storage/schema/delta/33/user_ips_index.sql | 17 ----------------- tests/handlers/test_device.py | 1 + tests/unittest.py | 9 ++++++++- 7 files changed, 36 insertions(+), 25 deletions(-) delete mode 100644 synapse/storage/schema/delta/33/user_ips_index.sql diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index edb1613a569f..47e0d9bf3aba 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -654,14 +654,16 @@ def _simple_upsert_txn_native_upsert( self, txn, table, keyvalues, values, insertion_values={} ): """ - Use the native UPSERT functionality in recent PostgreSQL and SQLite versions. + Use the native UPSERT functionality in recent PostgreSQL versions. """ allvalues = {} allvalues.update(keyvalues) allvalues.update(values) allvalues.update(insertion_values) - sql = "INSERT INTO %s (%s) VALUES (%s) ON CONFLICT (%s) DO UPDATE SET %s" % ( + sql = ("INSERT INTO %s (%s) VALUES (%s) " + "ON CONFLICT (%s) DO UPDATE SET %s " + "RETURNING (xmax = 0) AS inserted") % ( table, ", ".join(k for k in allvalues), ", ".join("?" for _ in allvalues), @@ -669,7 +671,9 @@ def _simple_upsert_txn_native_upsert( ", ".join(k + "=EXCLUDED." + k for k in values), ) txn.execute(sql, list(allvalues.values())) - return True + # One-tuple, which is a boolean for insertion or not + res = txn.fetchone() + return res[0] def _simple_select_one(self, table, keyvalues, retcols, allow_none=False, desc="_simple_select_one"): diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index 9ad17b7c2597..1c3aa4bff271 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -65,6 +65,15 @@ def __init__(self, db_conn, hs): columns=["last_seen"], ) + # Register a unique index + self.register_background_index_update( + "user_ips_device_unique_index", + index_name="user_ips_device_unique_id", + table="user_ips", + columns=["user_id", "access_token", "ip", "user_agent", "device_id"], + unique=True, + ) + # (user_id, access_token, ip) -> (user_agent, device_id, last_seen) self._batch_row_update = {} diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py index a4d16313902d..50b80da3261f 100644 --- a/synapse/storage/engines/sqlite.py +++ b/synapse/storage/engines/sqlite.py @@ -33,10 +33,17 @@ def __init__(self, database_module, database_config): @property def can_native_upsert(self): """ - Do we support native UPSERTs? This requires SQLite3 3.24+. + Do we support native UPSERTs? This requires SQLite3 3.24+, plus some + more work we haven't done yet to tell what was inserted vs updated. """ - from sqlite3 import sqlite_version_info - return sqlite_version_info >= (3, 24, 0) + # SQLite currently doesn't have a way of telling if an UPSERT ended up + # with an INSERT or an UPDATE + # To enable, uncomment: + # + # from sqlite3 import sqlite_version_info + # return sqlite_version_info >= (3, 24, 0) + + return False def check_database(self, txn): pass diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index fa36daac524d..e042221774bf 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -25,7 +25,7 @@ # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 53 +SCHEMA_VERSION = 54 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/schema/delta/33/user_ips_index.sql b/synapse/storage/schema/delta/33/user_ips_index.sql deleted file mode 100644 index 473f75a78e48..000000000000 --- a/synapse/storage/schema/delta/33/user_ips_index.sql +++ /dev/null @@ -1,17 +0,0 @@ -/* Copyright 2016 OpenMarket Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -INSERT INTO background_updates (update_name, progress_json) VALUES - ('user_ips_device_index', '{}'); diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index a3aa0a1cf240..eb1df21f5f18 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -82,6 +82,7 @@ def test_device_id_is_made_up_if_unspecified(self): dev = self.get_success(self.handler.store.get_device("@theresa:foo", device_id)) self.assertEqual(dev["display_name"], "display") + @unittest.INFO def test_get_devices_by_user(self): self._record_users() diff --git a/tests/unittest.py b/tests/unittest.py index 78d2f740f917..7d0fd1731903 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -333,7 +333,14 @@ def setup_test_homeserver(self, *args, **kwargs): """ kwargs = dict(kwargs) kwargs.update(self._hs_args) - return setup_test_homeserver(self.addCleanup, *args, **kwargs) + hs = setup_test_homeserver(self.addCleanup, *args, **kwargs) + stor = hs.get_datastore() + + # Run the database background updates. + while not self.get_success(stor.has_completed_background_updates()): + self.get_success(stor.do_next_background_update(1)) + + return hs def pump(self, by=0.0): """ From 1b287937c4c6df58217dacee374968984d2ce2fb Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 27 Dec 2018 23:23:28 +1100 Subject: [PATCH 18/40] add method to block upserts until we have built indexes --- synapse/storage/_base.py | 7 ++++++- synapse/storage/background_updates.py | 4 ++++ synapse/storage/client_ips.py | 3 ++- .../storage/schema/delta/33/user_ips_index.sql | 17 +++++++++++++++++ .../storage/schema/delta/54/user_ips_index.sql | 18 ++++++++++++++++++ tests/handlers/test_device.py | 1 - 6 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 synapse/storage/schema/delta/33/user_ips_index.sql create mode 100644 synapse/storage/schema/delta/54/user_ips_index.sql diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 47e0d9bf3aba..59886578745c 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -190,6 +190,11 @@ def __init__(self, db_conn, hs): self._pending_ds = [] + # We force simple upserts always, and then update in + # `has_completed_background_updates`, when any required indexes are + # built. + self._force_simple_upsert = True + self.database_engine = hs.database_engine def start_profiling(self): @@ -601,7 +606,7 @@ def _simple_upsert_txn( Pick the UPSERT method which works best on the platform. Either the native one (Pg9.5+, recent SQLites), or fall back to an emulated method. """ - if self.database_engine.can_native_upsert: + if self.database_engine.can_native_upsert and not self._force_simple_upsert: return self._simple_upsert_txn_native_upsert( txn, table, keyvalues, values, insertion_values=insertion_values ) diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 5fe1ca2de7f6..fa9d9de60e9b 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -143,6 +143,10 @@ def has_completed_background_updates(self): ) if not updates: self._all_done = True + + # Now that indexes are built, we are allowed to do native UPSERTs if + # the underlying database supports it. + self._force_simple_upsert = False defer.returnValue(True) defer.returnValue(False) diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index 1c3aa4bff271..7422bac0e7f4 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -123,7 +123,8 @@ def update(): ) def _update_client_ips_batch_txn(self, txn, to_update): - self.database_engine.lock_table(txn, "user_ips") + if self._force_simple_upsert or (not self.database_engine.can_native_upsert): + self.database_engine.lock_table(txn, "user_ips") for entry in iteritems(to_update): (user_id, access_token, ip), (user_agent, device_id, last_seen) = entry diff --git a/synapse/storage/schema/delta/33/user_ips_index.sql b/synapse/storage/schema/delta/33/user_ips_index.sql new file mode 100644 index 000000000000..473f75a78e48 --- /dev/null +++ b/synapse/storage/schema/delta/33/user_ips_index.sql @@ -0,0 +1,17 @@ +/* Copyright 2016 OpenMarket Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +INSERT INTO background_updates (update_name, progress_json) VALUES + ('user_ips_device_index', '{}'); diff --git a/synapse/storage/schema/delta/54/user_ips_index.sql b/synapse/storage/schema/delta/54/user_ips_index.sql new file mode 100644 index 000000000000..292bcbe8e998 --- /dev/null +++ b/synapse/storage/schema/delta/54/user_ips_index.sql @@ -0,0 +1,18 @@ +/* Copyright 2018 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- add a new unique index to user_ips table +INSERT INTO background_updates (update_name, progress_json) VALUES + ('user_ips_device_unique_index', '{}'); diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index eb1df21f5f18..a3aa0a1cf240 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -82,7 +82,6 @@ def test_device_id_is_made_up_if_unspecified(self): dev = self.get_success(self.handler.store.get_device("@theresa:foo", device_id)) self.assertEqual(dev["display_name"], "display") - @unittest.INFO def test_get_devices_by_user(self): self._record_users() From c97601fe5d4bcebc9f150a0b3f651d7d7afd0899 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 27 Dec 2018 23:46:07 +1100 Subject: [PATCH 19/40] redo best effort --- synapse/storage/_base.py | 51 ++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 59886578745c..fb3e23461c70 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -561,33 +561,12 @@ def _simple_upsert( values, insertion_values, lock=lock, + best_effort=best_effort, ) defer.returnValue(result) except ( self.database_engine.module.IntegrityError, - self.database_engine.module.OperationalError, ) as e: - - if isinstance(e, self.database_engine.module.OperationalError): - # We only care about serialization errors, so check for it - if ( - e.args[0] - == "could not serialize access due to concurrent update" - ): - # A concurrent update problem is when we try and do a - # native UPSERT but the row has changed from under us. - # We can either retry, or give up if asked to do so. - if best_effort: - # If it's a concurrent-update problem, and this is - # marked as 'best effort' (i.e. if there's a race, - # then the one we raced with will suffice), then - # pretend that we succeeded. - defer.returnValue(False) - else: - # Otherwise, raise, because it's a real OperationalError - # and we will need to be rolled back and retried. - raise - attempts += 1 if attempts >= 5: # don't retry forever, because things other than races @@ -600,7 +579,7 @@ def _simple_upsert( ) def _simple_upsert_txn( - self, txn, table, keyvalues, values, insertion_values={}, lock=True + self, txn, table, keyvalues, values, insertion_values={}, lock=True, best_effort=False, ): """ Pick the UPSERT method which works best on the platform. Either the @@ -608,7 +587,7 @@ def _simple_upsert_txn( """ if self.database_engine.can_native_upsert and not self._force_simple_upsert: return self._simple_upsert_txn_native_upsert( - txn, table, keyvalues, values, insertion_values=insertion_values + txn, table, keyvalues, values, insertion_values=insertion_values, best_effort=best_effort, ) else: return self._simple_upsert_txn_emulated( @@ -656,7 +635,7 @@ def _simple_upsert_txn_emulated( return True def _simple_upsert_txn_native_upsert( - self, txn, table, keyvalues, values, insertion_values={} + self, txn, table, keyvalues, values, insertion_values={}, best_effort=False, ): """ Use the native UPSERT functionality in recent PostgreSQL versions. @@ -675,7 +654,27 @@ def _simple_upsert_txn_native_upsert( ", ".join(k for k in keyvalues), ", ".join(k + "=EXCLUDED." + k for k in values), ) - txn.execute(sql, list(allvalues.values())) + try: + txn.execute(sql, list(allvalues.values())) + except self.database_engine.module.OperationalError as e: + # We only care about serialization errors, so check for it + if ( + e.args[0] + == "could not serialize access due to concurrent update" + ): + # A concurrent update problem is when we try and do a native + # UPSERT but the row has changed from under us. We can either + # retry, or give up if asked to do so. + if best_effort: + # If it's a concurrent-update problem, and this is marked as + # 'best effort' (i.e. if there's a race, then the one we + # raced with will suffice), then pretend that we succeeded. + return False + else: + # Otherwise, raise, because it's a real OperationalError and we + # will need to be rolled back and retried. + raise + # One-tuple, which is a boolean for insertion or not res = txn.fetchone() return res[0] From 02a52c9e346e595c4e6b447e927f82474851a9f6 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 27 Dec 2018 23:47:03 +1100 Subject: [PATCH 20/40] cleanups --- synapse/storage/_base.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index fb3e23461c70..496c0af7ecb1 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -564,9 +564,7 @@ def _simple_upsert( best_effort=best_effort, ) defer.returnValue(result) - except ( - self.database_engine.module.IntegrityError, - ) as e: + except self.database_engine.module.IntegrityError as e: attempts += 1 if attempts >= 5: # don't retry forever, because things other than races @@ -579,7 +577,14 @@ def _simple_upsert( ) def _simple_upsert_txn( - self, txn, table, keyvalues, values, insertion_values={}, lock=True, best_effort=False, + self, + txn, + table, + keyvalues, + values, + insertion_values={}, + lock=True, + best_effort=False, ): """ Pick the UPSERT method which works best on the platform. Either the @@ -587,7 +592,12 @@ def _simple_upsert_txn( """ if self.database_engine.can_native_upsert and not self._force_simple_upsert: return self._simple_upsert_txn_native_upsert( - txn, table, keyvalues, values, insertion_values=insertion_values, best_effort=best_effort, + txn, + table, + keyvalues, + values, + insertion_values=insertion_values, + best_effort=best_effort, ) else: return self._simple_upsert_txn_emulated( @@ -635,7 +645,7 @@ def _simple_upsert_txn_emulated( return True def _simple_upsert_txn_native_upsert( - self, txn, table, keyvalues, values, insertion_values={}, best_effort=False, + self, txn, table, keyvalues, values, insertion_values={}, best_effort=False ): """ Use the native UPSERT functionality in recent PostgreSQL versions. @@ -645,9 +655,11 @@ def _simple_upsert_txn_native_upsert( allvalues.update(values) allvalues.update(insertion_values) - sql = ("INSERT INTO %s (%s) VALUES (%s) " - "ON CONFLICT (%s) DO UPDATE SET %s " - "RETURNING (xmax = 0) AS inserted") % ( + sql = ( + "INSERT INTO %s (%s) VALUES (%s) " + "ON CONFLICT (%s) DO UPDATE SET %s " + "RETURNING (xmax = 0) AS inserted" + ) % ( table, ", ".join(k for k in allvalues), ", ".join("?" for _ in allvalues), @@ -658,10 +670,7 @@ def _simple_upsert_txn_native_upsert( txn.execute(sql, list(allvalues.values())) except self.database_engine.module.OperationalError as e: # We only care about serialization errors, so check for it - if ( - e.args[0] - == "could not serialize access due to concurrent update" - ): + if e.args[0] == "could not serialize access due to concurrent update": # A concurrent update problem is when we try and do a native # UPSERT but the row has changed from under us. We can either # retry, or give up if asked to do so. From 6272d0c424dcb967dac47dacbe489782340a940f Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 27 Dec 2018 23:51:06 +1100 Subject: [PATCH 21/40] respect blocked native upsert --- synapse/storage/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 496c0af7ecb1..fc9e7ee93a62 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -539,7 +539,7 @@ def _simple_upsert( try: result = yield self.runInteraction( desc, - self._simple_upsert_txn_native_upsert, + self._simple_upsert_txn, table, keyvalues, values, From 6233abf50ac7824c4fdb7176b1f49eff55e081b7 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 27 Dec 2018 23:51:51 +1100 Subject: [PATCH 22/40] respect blocked native upsert --- synapse/storage/_base.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index fc9e7ee93a62..6cf607483e16 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -533,23 +533,6 @@ def _simple_upsert( Deferred(bool): True if a new entry was created, False if an existing one was updated. """ - if self.database_engine.can_native_upsert: - # We don't put this in a loop as it is guaranteed to be atomic, so - # if we get an IntegrityError, it's unrelated. - try: - result = yield self.runInteraction( - desc, - self._simple_upsert_txn, - table, - keyvalues, - values, - insertion_values, - ) - defer.returnValue(result) - except Exception as e: - logger.warn("Got a %s, %s", (repr(e), str(e.args))) - raise - attempts = 0 while True: try: From a912b6dc965a2f6e64dfb8e6988de2c7659c6f3b Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 28 Dec 2018 00:47:34 +1100 Subject: [PATCH 23/40] move this so that slaves can safely upsert too --- synapse/storage/_base.py | 55 +++++++++++++++++++++++++-- synapse/storage/background_updates.py | 4 -- tests/unittest.py | 5 ++- 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 6cf607483e16..decc7de13464 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -190,12 +190,59 @@ def __init__(self, db_conn, hs): self._pending_ds = [] - # We force simple upserts always, and then update in - # `has_completed_background_updates`, when any required indexes are - # built. + self.database_engine = hs.database_engine + + # We force simple upserts by default, and then enable them once we have + # finished background updates (checked by `_check_safe_to_upsert`). self._force_simple_upsert = True - self.database_engine = hs.database_engine + # Check ASAP (and then later, every 1s) to see if we have finished + # background updates. + hs.get_reactor().callLater(0.0, self._check_safe_to_upsert) + + @defer.inlineCallbacks + def _check_safe_to_upsert(self): + """ + Is it safe to use native UPSERT? + + If there are background updates, we will need to wait, as they may be + the addition of indexes that set the UNIQUE constraint that we require. + + If the background updates have not completed, wait a second and check again. + """ + completed = yield self.has_completed_background_updates() + + if completed: + # Now that indexes are built, we are allowed to do native UPSERTs if + # the underlying database supports it. + self._force_simple_upsert = False + else: + self.hs.get_reactor().callLater(1.0, self._check_safe_to_upsert) + + + @defer.inlineCallbacks + def has_completed_background_updates(self): + """ + Check if all the background updates have completed. This is safe to run + on the master as well as slaves, and will be overridden by + background_updates.BackgroundUpdateStore if we are the master and have a + subclass with it. + + Returns: Deferred[bool]: True if all background updates have completed + """ + # otherwise, check if there are updates to be run. This is important, + # as we may be running on a worker which doesn't perform the bg updates + # itself, but still wants to wait for them to happen. + updates = yield self._simple_select_onecol( + "background_updates", + keyvalues=None, + retcol="1", + desc="check_background_updates", + ) + if not updates: + defer.returnValue(True) + + defer.returnValue(False) def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index fa9d9de60e9b..5fe1ca2de7f6 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -143,10 +143,6 @@ def has_completed_background_updates(self): ) if not updates: self._all_done = True - - # Now that indexes are built, we are allowed to do native UPSERTs if - # the underlying database supports it. - self._force_simple_upsert = False defer.returnValue(True) defer.returnValue(False) diff --git a/tests/unittest.py b/tests/unittest.py index 7d0fd1731903..6589196d855a 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -337,8 +337,9 @@ def setup_test_homeserver(self, *args, **kwargs): stor = hs.get_datastore() # Run the database background updates. - while not self.get_success(stor.has_completed_background_updates()): - self.get_success(stor.do_next_background_update(1)) + if hasattr(stor, "do_next_background_update"): + while not self.get_success(stor.has_completed_background_updates()): + self.get_success(stor.do_next_background_update(1)) return hs From 48c351ea5dcb7e71c3d39e63a96a64e0744e1bd5 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 28 Dec 2018 01:04:37 +1100 Subject: [PATCH 24/40] fix failing tests --- synapse/storage/_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index decc7de13464..120f5beb7db3 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -198,7 +198,7 @@ def __init__(self, db_conn, hs): # Check ASAP (and then later, every 1s) to see if we have finished # background updates. - hs.get_reactor().callLater(0.0, self._check_safe_to_upsert) + self._clock.call_later(0.0, self._check_safe_to_upsert) @defer.inlineCallbacks def _check_safe_to_upsert(self): @@ -217,7 +217,7 @@ def _check_safe_to_upsert(self): # the underlying database supports it. self._force_simple_upsert = False else: - self.hs.get_reactor().callLater(1.0, self._check_safe_to_upsert) + self._clock.call_later(1.0, self._check_safe_to_upsert) @defer.inlineCallbacks From c79058f68f98166c3cc406c2ca3c2156bd1705f8 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 28 Dec 2018 01:08:17 +1100 Subject: [PATCH 25/40] fix lint --- synapse/storage/_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 120f5beb7db3..ce074d678c8b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -219,7 +219,6 @@ def _check_safe_to_upsert(self): else: self._clock.call_later(1.0, self._check_safe_to_upsert) - @defer.inlineCallbacks def has_completed_background_updates(self): """ From ee4a0d858db11632cbc3ff8ed6da02ae815bdfe5 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 28 Dec 2018 01:16:54 +1100 Subject: [PATCH 26/40] fix failing tests --- synapse/storage/_base.py | 7 ++++--- tests/rest/client/v1/test_register.py | 11 +++++++---- tests/storage/test_base.py | 1 + tests/test_server.py | 12 +++++++++--- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index ce074d678c8b..89c1389029e1 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -196,9 +196,10 @@ def __init__(self, db_conn, hs): # finished background updates (checked by `_check_safe_to_upsert`). self._force_simple_upsert = True - # Check ASAP (and then later, every 1s) to see if we have finished - # background updates. - self._clock.call_later(0.0, self._check_safe_to_upsert) + if getattr(hs.config, "_enable_native_upserts", True): + # Check ASAP (and then later, every 1s) to see if we have finished + # background updates. + self._clock.call_later(0.0, self._check_safe_to_upsert) @defer.inlineCallbacks def _check_safe_to_upsert(self): diff --git a/tests/rest/client/v1/test_register.py b/tests/rest/client/v1/test_register.py index f973eff8cf8a..3d4f031e9afe 100644 --- a/tests/rest/client/v1/test_register.py +++ b/tests/rest/client/v1/test_register.py @@ -18,14 +18,17 @@ from mock import Mock from six import PY3 -from twisted.test.proto_helpers import MemoryReactorClock - from synapse.http.server import JsonResource from synapse.rest.client.v1_only.register import register_servlets from synapse.util import Clock from tests import unittest -from tests.server import make_request, render, setup_test_homeserver +from tests.server import ( + ThreadedMemoryReactorClock, + make_request, + render, + setup_test_homeserver, +) class CreateUserServletTestCase(unittest.TestCase): @@ -45,7 +48,7 @@ def setUp(self): ) handlers = Mock(registration_handler=self.registration_handler) - self.reactor = MemoryReactorClock() + self.reactor = ThreadedMemoryReactorClock() self.hs_clock = Clock(self.reactor) self.hs = self.hs = setup_test_homeserver( diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index 829f47d2e8a1..452d76ddd51e 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -49,6 +49,7 @@ def runWithConnection(func, *args, **kwargs): self.db_pool.runWithConnection = runWithConnection config = Mock() + config._enable_native_upserts = False config.event_cache_size = 1 config.database_config = {"name": "sqlite3"} hs = TestHomeServer( diff --git a/tests/test_server.py b/tests/test_server.py index 634a8fbca507..08fb3fe02f27 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -19,7 +19,7 @@ from twisted.internet.defer import Deferred from twisted.python.failure import Failure -from twisted.test.proto_helpers import AccumulatingProtocol, MemoryReactorClock +from twisted.test.proto_helpers import AccumulatingProtocol from twisted.web.resource import Resource from twisted.web.server import NOT_DONE_YET @@ -30,12 +30,18 @@ from synapse.util.logcontext import make_deferred_yieldable from tests import unittest -from tests.server import FakeTransport, make_request, render, setup_test_homeserver +from tests.server import ( + FakeTransport, + ThreadedMemoryReactorClock, + make_request, + render, + setup_test_homeserver, +) class JsonResourceTests(unittest.TestCase): def setUp(self): - self.reactor = MemoryReactorClock() + self.reactor = ThreadedMemoryReactorClock() self.hs_clock = Clock(self.reactor) self.homeserver = setup_test_homeserver( self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.reactor From 7df76f7bca512c86422cc0272c81d073425a3d53 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 28 Dec 2018 16:23:05 +1100 Subject: [PATCH 27/40] Update 4306.misc --- changelog.d/4306.misc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/4306.misc b/changelog.d/4306.misc index 58130b61908b..7f48b02fbf96 100644 --- a/changelog.d/4306.misc +++ b/changelog.d/4306.misc @@ -1 +1 @@ -Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+ and SQLite 3.24+. +Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+. From cd26a90c75034ad70ac123a120e2563556af7166 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 4 Jan 2019 03:22:14 +1100 Subject: [PATCH 28/40] add some full schemas + docs --- .travis.yml | 3 + synapse/storage/_base.py | 49 +- synapse/storage/client_ips.py | 2 +- synapse/storage/engines/postgres.py | 11 +- synapse/storage/event_push_actions.py | 20 +- synapse/storage/prepare_database.py | 14 +- .../schema/full_schemas/54/full.sql.postgres | 3186 +++++++++++++++++ .../schema/full_schemas/54/full.sql.sqlite | 1203 +++++++ .../storage/schema/full_schemas/README.txt | 16 + tests/unittest.py | 2 +- 10 files changed, 4455 insertions(+), 51 deletions(-) create mode 100644 synapse/storage/schema/full_schemas/54/full.sql.postgres create mode 100644 synapse/storage/schema/full_schemas/54/full.sql.sqlite create mode 100644 synapse/storage/schema/full_schemas/README.txt diff --git a/.travis.yml b/.travis.yml index 84d5efff9b00..728f0e248abf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,9 @@ cache: # - $HOME/.cache/pip/wheels +addons: + postgresql: "9.4" + # don't clone the whole repo history, one commit will do git: depth: 1 diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 89c1389029e1..50606f8675a2 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -192,13 +192,12 @@ def __init__(self, db_conn, hs): self.database_engine = hs.database_engine - # We force simple upserts by default, and then enable them once we have - # finished background updates (checked by `_check_safe_to_upsert`). - self._force_simple_upsert = True + # A set of tables that are not safe to use native upserts in. + self._unsafe_to_upsert_tables = {"user_ips"} - if getattr(hs.config, "_enable_native_upserts", True): + if self.database_engine.can_native_upsert: # Check ASAP (and then later, every 1s) to see if we have finished - # background updates. + # background updates of tables that aren't safe to update. self._clock.call_later(0.0, self._check_safe_to_upsert) @defer.inlineCallbacks @@ -211,38 +210,22 @@ def _check_safe_to_upsert(self): If the background updates have not completed, wait a second and check again. """ - completed = yield self.has_completed_background_updates() - - if completed: - # Now that indexes are built, we are allowed to do native UPSERTs if - # the underlying database supports it. - self._force_simple_upsert = False - else: - self._clock.call_later(1.0, self._check_safe_to_upsert) - - @defer.inlineCallbacks - def has_completed_background_updates(self): - """ - Check if all the background updates have completed. This is safe to run - on the master as well as slaves, and will be overridden by - background_updates.BackgroundUpdateStore if we are the master and have a - subclass with it. - - Returns: Deferred[bool]: True if all background updates have completed - """ - # otherwise, check if there are updates to be run. This is important, - # as we may be running on a worker which doesn't perform the bg updates - # itself, but still wants to wait for them to happen. - updates = yield self._simple_select_onecol( + updates = yield self._simple_select_list( "background_updates", keyvalues=None, - retcol="1", + retcols=["update_name"], desc="check_background_updates", ) - if not updates: - defer.returnValue(True) + updates = [x["update_name"] for x in updates] - defer.returnValue(False) + # The User IPs table in schema #53 was missing a unique index, which we + # run as a background update. + if not "user_ips_device_unique_index" in updates: + self._unsafe_to_upsert_tables.discard("user_id") + + # If there's any tables left to check, reschedule to run. + if self._unsafe_to_upsert_tables: + self._clock.call_later(1.0, self._check_safe_to_upsert) def start_profiling(self): self._previous_loop_ts = self._clock.time_msec() @@ -620,7 +603,7 @@ def _simple_upsert_txn( Pick the UPSERT method which works best on the platform. Either the native one (Pg9.5+, recent SQLites), or fall back to an emulated method. """ - if self.database_engine.can_native_upsert and not self._force_simple_upsert: + if self.database_engine.can_native_upsert and table not in self._unsafe_to_upsert_tables: return self._simple_upsert_txn_native_upsert( txn, table, diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index 7422bac0e7f4..e4261d78cd6e 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -123,7 +123,7 @@ def update(): ) def _update_client_ips_batch_txn(self, txn, to_update): - if self._force_simple_upsert or (not self.database_engine.can_native_upsert): + if "user_ips" in self._unsafe_to_upsert_tables or (not self.database_engine.can_native_upsert): self.database_engine.lock_table(txn, "user_ips") for entry in iteritems(to_update): diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index e941608297ff..4004427c7b85 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -42,13 +42,8 @@ def on_new_connection(self, db_conn): # Get the version of PostgreSQL that we're using. As per the psycopg2 # docs: The number is formed by converting the major, minor, and # revision numbers into two-decimal-digit numbers and appending them - # together. For example, version 8.1.5 will be returned as 80105. - server_version = str(db_conn.server_version) - self._version = ( - int(server_version[:-4], 10), - int(server_version[-4:-2], 10), - int(server_version[-2:], 10) - ) + # together. For example, version 8.1.5 will be returned as 80105 + self._version = db_conn.server_version db_conn.set_isolation_level( self.module.extensions.ISOLATION_LEVEL_REPEATABLE_READ @@ -71,7 +66,7 @@ def can_native_upsert(self): """ Can we use native UPSERTs? This requires PostgreSQL 9.5+. """ - return self._version >= (9, 5, 0) + return self._version >= 90500 def is_deadlock(self, error): if isinstance(error, self.module.DatabaseError): diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 6840320641da..371e8f7f3eed 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -22,6 +22,7 @@ from twisted.internet import defer +from synapse.api.errors import StoreError from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage._base import LoggingTransaction, SQLBaseStore from synapse.util.caches.descriptors import cachedInlineCallbacks @@ -850,12 +851,19 @@ def _rotate_notifs_txn(self, txn): return caught_up def _rotate_notifs_before_txn(self, txn, rotate_to_stream_ordering): - old_rotate_stream_ordering = self._simple_select_one_onecol_txn( - txn, - table="event_push_summary_stream_ordering", - keyvalues={}, - retcol="stream_ordering", - ) + try: + old_rotate_stream_ordering = self._simple_select_one_onecol_txn( + txn, + table="event_push_summary_stream_ordering", + keyvalues={}, + retcol="stream_ordering", + ) + except StoreError as e: + if e.code == 404: + # The database is empty, assume 0. + old_rotate_stream_ordering = 0 + else: + raise # Calculate the new counts that should be upserted into event_push_summary sql = """ diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index e042221774bf..3d79c3d50fc6 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -20,6 +20,8 @@ import os import re +from synapse.storage.engines.postgres import PostgresEngine + logger = logging.getLogger(__name__) @@ -115,8 +117,16 @@ def _setup_new_database(cur, database_engine): valid_dirs = [] pattern = re.compile(r"^\d+(\.sql)?$") + + if isinstance(database_engine, PostgresEngine): + specific = "postgres" + else: + specific = "sqlite" + + specific_pattern = re.compile(r"^\d+(\.sql." + specific + r")?$") + for filename in directory_entries: - match = pattern.match(filename) + match = pattern.match(filename) or specific_pattern.match(filename) abs_path = os.path.join(current_dir, filename) if match and os.path.isdir(abs_path): ver = int(match.group(0)) @@ -136,7 +146,7 @@ def _setup_new_database(cur, database_engine): directory_entries = os.listdir(sql_dir) - for filename in fnmatch.filter(directory_entries, "*.sql"): + for filename in fnmatch.filter(directory_entries, "*.sql") + fnmatch.filter(directory_entries, "*.sql." + specific): sql_loc = os.path.join(sql_dir, filename) logger.debug("Applying schema %s", sql_loc) executescript(cur, sql_loc) diff --git a/synapse/storage/schema/full_schemas/54/full.sql.postgres b/synapse/storage/schema/full_schemas/54/full.sql.postgres new file mode 100644 index 000000000000..245db5aff0b1 --- /dev/null +++ b/synapse/storage/schema/full_schemas/54/full.sql.postgres @@ -0,0 +1,3186 @@ +-- +-- PostgreSQL database dump +-- + +-- Dumped from database version 11.1 +-- Dumped by pg_dump version 11.1 + +-- +-- Name: access_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE access_tokens ( + id bigint NOT NULL, + user_id text NOT NULL, + device_id text, + token text NOT NULL, + last_used bigint +); + + +-- +-- Name: account_data; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE account_data ( + user_id text NOT NULL, + account_data_type text NOT NULL, + stream_id bigint NOT NULL, + content text NOT NULL +); + + +-- +-- Name: account_data_max_stream_id; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE account_data_max_stream_id ( + lock character(1) DEFAULT 'X'::bpchar NOT NULL, + stream_id bigint NOT NULL, + CONSTRAINT private_user_data_max_stream_id_lock_check CHECK ((lock = 'X'::bpchar)) +); + + +-- +-- Name: application_services; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE application_services ( + id bigint NOT NULL, + url text, + token text, + hs_token text, + sender text +); + + +-- +-- Name: application_services_regex; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE application_services_regex ( + id bigint NOT NULL, + as_id bigint NOT NULL, + namespace integer, + regex text +); + + +-- +-- Name: application_services_state; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE application_services_state ( + as_id text NOT NULL, + state character varying(5), + last_txn integer +); + + +-- +-- Name: application_services_txns; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE application_services_txns ( + as_id text NOT NULL, + txn_id integer NOT NULL, + event_ids text NOT NULL +); + + +-- +-- Name: appservice_room_list; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE appservice_room_list ( + appservice_id text NOT NULL, + network_id text NOT NULL, + room_id text NOT NULL +); + + +-- +-- Name: appservice_stream_position; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE appservice_stream_position ( + lock character(1) DEFAULT 'X'::bpchar NOT NULL, + stream_ordering bigint, + CONSTRAINT appservice_stream_position_lock_check CHECK ((lock = 'X'::bpchar)) +); + + +-- +-- Name: background_updates; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE background_updates ( + update_name text NOT NULL, + progress_json text NOT NULL, + depends_on text +); + + +-- +-- Name: blocked_rooms; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE blocked_rooms ( + room_id text NOT NULL, + user_id text NOT NULL +); + + +-- +-- Name: cache_invalidation_stream; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE cache_invalidation_stream ( + stream_id bigint, + cache_func text, + keys text[], + invalidation_ts bigint +); + + +-- +-- Name: current_state_delta_stream; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE current_state_delta_stream ( + stream_id bigint NOT NULL, + room_id text NOT NULL, + type text NOT NULL, + state_key text NOT NULL, + event_id text, + prev_event_id text +); + + +-- +-- Name: current_state_events; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE current_state_events ( + event_id text NOT NULL, + room_id text NOT NULL, + type text NOT NULL, + state_key text NOT NULL +); + + +-- +-- Name: current_state_resets; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE current_state_resets ( + event_stream_ordering bigint NOT NULL +); + + +-- +-- Name: deleted_pushers; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE deleted_pushers ( + stream_id bigint NOT NULL, + app_id text NOT NULL, + pushkey text NOT NULL, + user_id text NOT NULL +); + + +-- +-- Name: destinations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE destinations ( + destination text NOT NULL, + retry_last_ts bigint, + retry_interval integer +); + + +-- +-- Name: device_federation_inbox; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE device_federation_inbox ( + origin text NOT NULL, + message_id text NOT NULL, + received_ts bigint NOT NULL +); + + +-- +-- Name: device_federation_outbox; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE device_federation_outbox ( + destination text NOT NULL, + stream_id bigint NOT NULL, + queued_ts bigint NOT NULL, + messages_json text NOT NULL +); + + +-- +-- Name: device_inbox; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE device_inbox ( + user_id text NOT NULL, + device_id text NOT NULL, + stream_id bigint NOT NULL, + message_json text NOT NULL +); + + +-- +-- Name: device_lists_outbound_last_success; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE device_lists_outbound_last_success ( + destination text NOT NULL, + user_id text NOT NULL, + stream_id bigint NOT NULL +); + + +-- +-- Name: device_lists_outbound_pokes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE device_lists_outbound_pokes ( + destination text NOT NULL, + stream_id bigint NOT NULL, + user_id text NOT NULL, + device_id text NOT NULL, + sent boolean NOT NULL, + ts bigint NOT NULL +); + + +-- +-- Name: device_lists_remote_cache; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE device_lists_remote_cache ( + user_id text NOT NULL, + device_id text NOT NULL, + content text NOT NULL +); + + +-- +-- Name: device_lists_remote_extremeties; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE device_lists_remote_extremeties ( + user_id text NOT NULL, + stream_id text NOT NULL +); + + +-- +-- Name: device_lists_stream; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE device_lists_stream ( + stream_id bigint NOT NULL, + user_id text NOT NULL, + device_id text NOT NULL +); + + +-- +-- Name: device_max_stream_id; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE device_max_stream_id ( + stream_id bigint NOT NULL +); + + +-- +-- Name: devices; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE devices ( + user_id text NOT NULL, + device_id text NOT NULL, + display_name text +); + + +-- +-- Name: e2e_device_keys_json; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE e2e_device_keys_json ( + user_id text NOT NULL, + device_id text NOT NULL, + ts_added_ms bigint NOT NULL, + key_json text NOT NULL +); + + +-- +-- Name: e2e_one_time_keys_json; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE e2e_one_time_keys_json ( + user_id text NOT NULL, + device_id text NOT NULL, + algorithm text NOT NULL, + key_id text NOT NULL, + ts_added_ms bigint NOT NULL, + key_json text NOT NULL +); + + +-- +-- Name: e2e_room_keys; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE e2e_room_keys ( + user_id text NOT NULL, + room_id text NOT NULL, + session_id text NOT NULL, + version bigint NOT NULL, + first_message_index integer, + forwarded_count integer, + is_verified boolean, + session_data text NOT NULL +); + + +-- +-- Name: e2e_room_keys_versions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE e2e_room_keys_versions ( + user_id text NOT NULL, + version bigint NOT NULL, + algorithm text NOT NULL, + auth_data text NOT NULL, + deleted smallint DEFAULT 0 NOT NULL +); + + +-- +-- Name: erased_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE erased_users ( + user_id text NOT NULL +); + + +-- +-- Name: event_auth; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE event_auth ( + event_id text NOT NULL, + auth_id text NOT NULL, + room_id text NOT NULL +); + + +-- +-- Name: event_backward_extremities; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE event_backward_extremities ( + event_id text NOT NULL, + room_id text NOT NULL +); + + +-- +-- Name: event_content_hashes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE event_content_hashes ( + event_id text, + algorithm text, + hash bytea +); + + +-- +-- Name: event_destinations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE event_destinations ( + event_id text NOT NULL, + destination text NOT NULL, + delivered_ts bigint DEFAULT 0 +); + + +-- +-- Name: event_edge_hashes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE event_edge_hashes ( + event_id text, + prev_event_id text, + algorithm text, + hash bytea +); + + +-- +-- Name: event_edges; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE event_edges ( + event_id text NOT NULL, + prev_event_id text NOT NULL, + room_id text NOT NULL, + is_state boolean NOT NULL +); + + +-- +-- Name: event_forward_extremities; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE event_forward_extremities ( + event_id text NOT NULL, + room_id text NOT NULL +); + + +-- +-- Name: event_json; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE event_json ( + event_id text NOT NULL, + room_id text NOT NULL, + internal_metadata text NOT NULL, + json text NOT NULL +); + + +-- +-- Name: event_push_actions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE event_push_actions ( + room_id text NOT NULL, + event_id text NOT NULL, + user_id text NOT NULL, + profile_tag character varying(32), + actions text NOT NULL, + topological_ordering bigint, + stream_ordering bigint, + notif smallint, + highlight smallint +); + + +-- +-- Name: event_push_actions_staging; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE event_push_actions_staging ( + event_id text NOT NULL, + user_id text NOT NULL, + actions text NOT NULL, + notif smallint NOT NULL, + highlight smallint NOT NULL +); + + +-- +-- Name: event_push_summary; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE event_push_summary ( + user_id text NOT NULL, + room_id text NOT NULL, + notif_count bigint NOT NULL, + stream_ordering bigint NOT NULL +); + + +-- +-- Name: event_push_summary_stream_ordering; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE event_push_summary_stream_ordering ( + lock character(1) DEFAULT 'X'::bpchar NOT NULL, + stream_ordering bigint NOT NULL, + CONSTRAINT event_push_summary_stream_ordering_lock_check CHECK ((lock = 'X'::bpchar)) +); + + +-- +-- Name: event_reference_hashes; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE event_reference_hashes ( + event_id text, + algorithm text, + hash bytea +); + + +-- +-- Name: event_reports; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE event_reports ( + id bigint NOT NULL, + received_ts bigint NOT NULL, + room_id text NOT NULL, + event_id text NOT NULL, + user_id text NOT NULL, + reason text, + content text +); + + +-- +-- Name: event_search; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE event_search ( + event_id text, + room_id text, + sender text, + key text, + vector tsvector, + origin_server_ts bigint, + stream_ordering bigint +); + + +-- +-- Name: event_signatures; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE event_signatures ( + event_id text, + signature_name text, + key_id text, + signature bytea +); + + +-- +-- Name: event_to_state_groups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE event_to_state_groups ( + event_id text NOT NULL, + state_group bigint NOT NULL +); + + +-- +-- Name: events; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE events ( + stream_ordering integer NOT NULL, + topological_ordering bigint NOT NULL, + event_id text NOT NULL, + type text NOT NULL, + room_id text NOT NULL, + content text, + unrecognized_keys text, + processed boolean NOT NULL, + outlier boolean NOT NULL, + depth bigint DEFAULT 0 NOT NULL, + origin_server_ts bigint, + received_ts bigint, + sender text, + contains_url boolean +); + + +-- +-- Name: ex_outlier_stream; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE ex_outlier_stream ( + event_stream_ordering bigint NOT NULL, + event_id text NOT NULL, + state_group bigint NOT NULL +); + + +-- +-- Name: federation_stream_position; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE federation_stream_position ( + type text NOT NULL, + stream_id integer NOT NULL +); + + +-- +-- Name: feedback; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE feedback ( + event_id text NOT NULL, + feedback_type text, + target_event_id text, + sender text, + room_id text +); + + +-- +-- Name: group_attestations_remote; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE group_attestations_remote ( + group_id text NOT NULL, + user_id text NOT NULL, + valid_until_ms bigint NOT NULL, + attestation_json text NOT NULL +); + + +-- +-- Name: group_attestations_renewals; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE group_attestations_renewals ( + group_id text NOT NULL, + user_id text NOT NULL, + valid_until_ms bigint NOT NULL +); + + +-- +-- Name: group_invites; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE group_invites ( + group_id text NOT NULL, + user_id text NOT NULL +); + + +-- +-- Name: group_roles; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE group_roles ( + group_id text NOT NULL, + role_id text NOT NULL, + profile text NOT NULL, + is_public boolean NOT NULL +); + + +-- +-- Name: group_room_categories; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE group_room_categories ( + group_id text NOT NULL, + category_id text NOT NULL, + profile text NOT NULL, + is_public boolean NOT NULL +); + + +-- +-- Name: group_rooms; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE group_rooms ( + group_id text NOT NULL, + room_id text NOT NULL, + is_public boolean NOT NULL +); + + +-- +-- Name: group_summary_roles; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE group_summary_roles ( + group_id text NOT NULL, + role_id text NOT NULL, + role_order bigint NOT NULL, + CONSTRAINT group_summary_roles_role_order_check CHECK ((role_order > 0)) +); + + +-- +-- Name: group_summary_room_categories; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE group_summary_room_categories ( + group_id text NOT NULL, + category_id text NOT NULL, + cat_order bigint NOT NULL, + CONSTRAINT group_summary_room_categories_cat_order_check CHECK ((cat_order > 0)) +); + + +-- +-- Name: group_summary_rooms; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE group_summary_rooms ( + group_id text NOT NULL, + room_id text NOT NULL, + category_id text NOT NULL, + room_order bigint NOT NULL, + is_public boolean NOT NULL, + CONSTRAINT group_summary_rooms_room_order_check CHECK ((room_order > 0)) +); + + +-- +-- Name: group_summary_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE group_summary_users ( + group_id text NOT NULL, + user_id text NOT NULL, + role_id text NOT NULL, + user_order bigint NOT NULL, + is_public boolean NOT NULL +); + + +-- +-- Name: group_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE group_users ( + group_id text NOT NULL, + user_id text NOT NULL, + is_admin boolean NOT NULL, + is_public boolean NOT NULL +); + + +-- +-- Name: groups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE groups ( + group_id text NOT NULL, + name text, + avatar_url text, + short_description text, + long_description text, + is_public boolean NOT NULL, + join_policy text DEFAULT 'invite'::text NOT NULL +); + + +-- +-- Name: guest_access; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE guest_access ( + event_id text NOT NULL, + room_id text NOT NULL, + guest_access text NOT NULL +); + + +-- +-- Name: history_visibility; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE history_visibility ( + event_id text NOT NULL, + room_id text NOT NULL, + history_visibility text NOT NULL +); + + +-- +-- Name: local_group_membership; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE local_group_membership ( + group_id text NOT NULL, + user_id text NOT NULL, + is_admin boolean NOT NULL, + membership text NOT NULL, + is_publicised boolean NOT NULL, + content text NOT NULL +); + + +-- +-- Name: local_group_updates; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE local_group_updates ( + stream_id bigint NOT NULL, + group_id text NOT NULL, + user_id text NOT NULL, + type text NOT NULL, + content text NOT NULL +); + + +-- +-- Name: local_invites; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE local_invites ( + stream_id bigint NOT NULL, + inviter text NOT NULL, + invitee text NOT NULL, + event_id text NOT NULL, + room_id text NOT NULL, + locally_rejected text, + replaced_by text +); + + +-- +-- Name: local_media_repository; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE local_media_repository ( + media_id text, + media_type text, + media_length integer, + created_ts bigint, + upload_name text, + user_id text, + quarantined_by text, + url_cache text, + last_access_ts bigint +); + + +-- +-- Name: local_media_repository_thumbnails; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE local_media_repository_thumbnails ( + media_id text, + thumbnail_width integer, + thumbnail_height integer, + thumbnail_type text, + thumbnail_method text, + thumbnail_length integer +); + + +-- +-- Name: local_media_repository_url_cache; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE local_media_repository_url_cache ( + url text, + response_code integer, + etag text, + expires_ts bigint, + og text, + media_id text, + download_ts bigint +); + + +-- +-- Name: monthly_active_users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE monthly_active_users ( + user_id text NOT NULL, + "timestamp" bigint NOT NULL +); + + +-- +-- Name: open_id_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE open_id_tokens ( + token text NOT NULL, + ts_valid_until_ms bigint NOT NULL, + user_id text NOT NULL +); + + +-- +-- Name: presence; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE presence ( + user_id text NOT NULL, + state character varying(20), + status_msg text, + mtime bigint +); + + +-- +-- Name: presence_allow_inbound; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE presence_allow_inbound ( + observed_user_id text NOT NULL, + observer_user_id text NOT NULL +); + + +-- +-- Name: presence_list; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE presence_list ( + user_id text NOT NULL, + observed_user_id text NOT NULL, + accepted boolean NOT NULL +); + + +-- +-- Name: presence_stream; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE presence_stream ( + stream_id bigint, + user_id text, + state text, + last_active_ts bigint, + last_federation_update_ts bigint, + last_user_sync_ts bigint, + status_msg text, + currently_active boolean +); + + +-- +-- Name: profiles; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE profiles ( + user_id text NOT NULL, + displayname text, + avatar_url text +); + + +-- +-- Name: public_room_list_stream; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public_room_list_stream ( + stream_id bigint NOT NULL, + room_id text NOT NULL, + visibility boolean NOT NULL, + appservice_id text, + network_id text +); + + +-- +-- Name: push_rules; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE push_rules ( + id bigint NOT NULL, + user_name text NOT NULL, + rule_id text NOT NULL, + priority_class smallint NOT NULL, + priority integer DEFAULT 0 NOT NULL, + conditions text NOT NULL, + actions text NOT NULL +); + + +-- +-- Name: push_rules_enable; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE push_rules_enable ( + id bigint NOT NULL, + user_name text NOT NULL, + rule_id text NOT NULL, + enabled smallint +); + + +-- +-- Name: push_rules_stream; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE push_rules_stream ( + stream_id bigint NOT NULL, + event_stream_ordering bigint NOT NULL, + user_id text NOT NULL, + rule_id text NOT NULL, + op text NOT NULL, + priority_class smallint, + priority integer, + conditions text, + actions text +); + + +-- +-- Name: pusher_throttle; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE pusher_throttle ( + pusher bigint NOT NULL, + room_id text NOT NULL, + last_sent_ts bigint, + throttle_ms bigint +); + + +-- +-- Name: pushers; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE pushers ( + id bigint NOT NULL, + user_name text NOT NULL, + access_token bigint, + profile_tag text NOT NULL, + kind text NOT NULL, + app_id text NOT NULL, + app_display_name text NOT NULL, + device_display_name text NOT NULL, + pushkey text NOT NULL, + ts bigint NOT NULL, + lang text, + data text, + last_stream_ordering integer, + last_success bigint, + failing_since bigint +); + + +-- +-- Name: ratelimit_override; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE ratelimit_override ( + user_id text NOT NULL, + messages_per_second bigint, + burst_count bigint +); + + +-- +-- Name: receipts_graph; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE receipts_graph ( + room_id text NOT NULL, + receipt_type text NOT NULL, + user_id text NOT NULL, + event_ids text NOT NULL, + data text NOT NULL +); + + +-- +-- Name: receipts_linearized; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE receipts_linearized ( + stream_id bigint NOT NULL, + room_id text NOT NULL, + receipt_type text NOT NULL, + user_id text NOT NULL, + event_id text NOT NULL, + data text NOT NULL +); + + +-- +-- Name: received_transactions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE received_transactions ( + transaction_id text, + origin text, + ts bigint, + response_code integer, + response_json bytea, + has_been_referenced smallint DEFAULT 0 +); + + +-- +-- Name: redactions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE redactions ( + event_id text NOT NULL, + redacts text NOT NULL +); + + +-- +-- Name: rejections; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE rejections ( + event_id text NOT NULL, + reason text NOT NULL, + last_check text NOT NULL +); + + +-- +-- Name: remote_media_cache; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE remote_media_cache ( + media_origin text, + media_id text, + media_type text, + created_ts bigint, + upload_name text, + media_length integer, + filesystem_id text, + last_access_ts bigint, + quarantined_by text +); + + +-- +-- Name: remote_media_cache_thumbnails; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE remote_media_cache_thumbnails ( + media_origin text, + media_id text, + thumbnail_width integer, + thumbnail_height integer, + thumbnail_method text, + thumbnail_type text, + thumbnail_length integer, + filesystem_id text +); + + +-- +-- Name: remote_profile_cache; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE remote_profile_cache ( + user_id text NOT NULL, + displayname text, + avatar_url text, + last_check bigint NOT NULL +); + + +-- +-- Name: room_account_data; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE room_account_data ( + user_id text NOT NULL, + room_id text NOT NULL, + account_data_type text NOT NULL, + stream_id bigint NOT NULL, + content text NOT NULL +); + + +-- +-- Name: room_alias_servers; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE room_alias_servers ( + room_alias text NOT NULL, + server text NOT NULL +); + + +-- +-- Name: room_aliases; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE room_aliases ( + room_alias text NOT NULL, + room_id text NOT NULL, + creator text +); + + +-- +-- Name: room_depth; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE room_depth ( + room_id text NOT NULL, + min_depth integer NOT NULL +); + + +-- +-- Name: room_hosts; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE room_hosts ( + room_id text NOT NULL, + host text NOT NULL +); + + +-- +-- Name: room_memberships; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE room_memberships ( + event_id text NOT NULL, + user_id text NOT NULL, + sender text NOT NULL, + room_id text NOT NULL, + membership text NOT NULL, + forgotten integer DEFAULT 0, + display_name text, + avatar_url text +); + + +-- +-- Name: room_names; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE room_names ( + event_id text NOT NULL, + room_id text NOT NULL, + name text NOT NULL +); + + +-- +-- Name: room_tags; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE room_tags ( + user_id text NOT NULL, + room_id text NOT NULL, + tag text NOT NULL, + content text NOT NULL +); + + +-- +-- Name: room_tags_revisions; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE room_tags_revisions ( + user_id text NOT NULL, + room_id text NOT NULL, + stream_id bigint NOT NULL +); + + +-- +-- Name: rooms; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE rooms ( + room_id text NOT NULL, + is_public boolean, + creator text +); + + +-- +-- Name: server_keys_json; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE server_keys_json ( + server_name text NOT NULL, + key_id text NOT NULL, + from_server text NOT NULL, + ts_added_ms bigint NOT NULL, + ts_valid_until_ms bigint NOT NULL, + key_json bytea NOT NULL +); + + +-- +-- Name: server_signature_keys; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE server_signature_keys ( + server_name text, + key_id text, + from_server text, + ts_added_ms bigint, + verify_key bytea +); + + +-- +-- Name: server_tls_certificates; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE server_tls_certificates ( + server_name text, + fingerprint text, + from_server text, + ts_added_ms bigint, + tls_certificate bytea +); + + +-- +-- Name: state_events; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE state_events ( + event_id text NOT NULL, + room_id text NOT NULL, + type text NOT NULL, + state_key text NOT NULL, + prev_state text +); + + +-- +-- Name: state_forward_extremities; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE state_forward_extremities ( + event_id text NOT NULL, + room_id text NOT NULL, + type text NOT NULL, + state_key text NOT NULL +); + + +-- +-- Name: state_group_edges; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE state_group_edges ( + state_group bigint NOT NULL, + prev_state_group bigint NOT NULL +); + + +-- +-- Name: state_group_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE state_group_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: state_groups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE state_groups ( + id bigint NOT NULL, + room_id text NOT NULL, + event_id text NOT NULL +); + + +-- +-- Name: state_groups_state; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE state_groups_state ( + state_group bigint NOT NULL, + room_id text NOT NULL, + type text NOT NULL, + state_key text NOT NULL, + event_id text NOT NULL +); + + +-- +-- Name: stats_reporting; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE stats_reporting ( + reported_stream_token integer, + reported_time bigint +); + + +-- +-- Name: stream_ordering_to_exterm; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE stream_ordering_to_exterm ( + stream_ordering bigint NOT NULL, + room_id text NOT NULL, + event_id text NOT NULL +); + + +-- +-- Name: threepid_guest_access_tokens; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE threepid_guest_access_tokens ( + medium text, + address text, + guest_access_token text, + first_inviter text +); + + +-- +-- Name: topics; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE topics ( + event_id text NOT NULL, + room_id text NOT NULL, + topic text NOT NULL +); + + +-- +-- Name: transaction_id_to_pdu; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE transaction_id_to_pdu ( + transaction_id integer, + destination text, + pdu_id text, + pdu_origin text +); + + +-- +-- Name: user_daily_visits; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE user_daily_visits ( + user_id text NOT NULL, + device_id text, + "timestamp" bigint NOT NULL +); + + +-- +-- Name: user_directory; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE user_directory ( + user_id text NOT NULL, + room_id text, + display_name text, + avatar_url text +); + + +-- +-- Name: user_directory_search; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE user_directory_search ( + user_id text NOT NULL, + vector tsvector +); + + +-- +-- Name: user_directory_stream_pos; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE user_directory_stream_pos ( + lock character(1) DEFAULT 'X'::bpchar NOT NULL, + stream_id bigint, + CONSTRAINT user_directory_stream_pos_lock_check CHECK ((lock = 'X'::bpchar)) +); + + +-- +-- Name: user_filters; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE user_filters ( + user_id text, + filter_id bigint, + filter_json bytea +); + + +-- +-- Name: user_ips; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE user_ips ( + user_id text NOT NULL, + access_token text NOT NULL, + device_id text, + ip text NOT NULL, + user_agent text NOT NULL, + last_seen bigint NOT NULL +); + + +-- +-- Name: user_threepids; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE user_threepids ( + user_id text NOT NULL, + medium text NOT NULL, + address text NOT NULL, + validated_at bigint NOT NULL, + added_at bigint NOT NULL +); + + +-- +-- Name: users; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE users ( + name text, + password_hash text, + creation_ts bigint, + admin smallint DEFAULT 0 NOT NULL, + upgrade_ts bigint, + is_guest smallint DEFAULT 0 NOT NULL, + appservice_id text, + consent_version text, + consent_server_notice_sent text, + user_type text +); + + +-- +-- Name: users_in_public_rooms; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE users_in_public_rooms ( + user_id text NOT NULL, + room_id text NOT NULL +); + + +-- +-- Name: users_pending_deactivation; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE users_pending_deactivation ( + user_id text NOT NULL +); + + +-- +-- Name: users_who_share_rooms; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE users_who_share_rooms ( + user_id text NOT NULL, + other_user_id text NOT NULL, + room_id text NOT NULL, + share_private boolean NOT NULL +); + + +-- +-- Name: access_tokens access_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY access_tokens + ADD CONSTRAINT access_tokens_pkey PRIMARY KEY (id); + + +-- +-- Name: access_tokens access_tokens_token_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY access_tokens + ADD CONSTRAINT access_tokens_token_key UNIQUE (token); + + +-- +-- Name: account_data account_data_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY account_data + ADD CONSTRAINT account_data_uniqueness UNIQUE (user_id, account_data_type); + + +-- +-- Name: application_services application_services_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY application_services + ADD CONSTRAINT application_services_pkey PRIMARY KEY (id); + + +-- +-- Name: application_services_regex application_services_regex_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY application_services_regex + ADD CONSTRAINT application_services_regex_pkey PRIMARY KEY (id); + + +-- +-- Name: application_services_state application_services_state_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY application_services_state + ADD CONSTRAINT application_services_state_pkey PRIMARY KEY (as_id); + + +-- +-- Name: application_services application_services_token_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY application_services + ADD CONSTRAINT application_services_token_key UNIQUE (token); + + +-- +-- Name: application_services_txns application_services_txns_as_id_txn_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY application_services_txns + ADD CONSTRAINT application_services_txns_as_id_txn_id_key UNIQUE (as_id, txn_id); + + +-- +-- Name: appservice_stream_position appservice_stream_position_lock_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY appservice_stream_position + ADD CONSTRAINT appservice_stream_position_lock_key UNIQUE (lock); + + +-- +-- Name: background_updates background_updates_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY background_updates + ADD CONSTRAINT background_updates_uniqueness UNIQUE (update_name); + + +-- +-- Name: current_state_events current_state_events_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY current_state_events + ADD CONSTRAINT current_state_events_event_id_key UNIQUE (event_id); + + +-- +-- Name: current_state_events current_state_events_room_id_type_state_key_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY current_state_events + ADD CONSTRAINT current_state_events_room_id_type_state_key_key UNIQUE (room_id, type, state_key); + + +-- +-- Name: current_state_resets current_state_resets_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY current_state_resets + ADD CONSTRAINT current_state_resets_pkey PRIMARY KEY (event_stream_ordering); + + +-- +-- Name: destinations destinations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY destinations + ADD CONSTRAINT destinations_pkey PRIMARY KEY (destination); + + +-- +-- Name: devices device_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY devices + ADD CONSTRAINT device_uniqueness UNIQUE (user_id, device_id); + + +-- +-- Name: e2e_device_keys_json e2e_device_keys_json_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY e2e_device_keys_json + ADD CONSTRAINT e2e_device_keys_json_uniqueness UNIQUE (user_id, device_id); + + +-- +-- Name: e2e_one_time_keys_json e2e_one_time_keys_json_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY e2e_one_time_keys_json + ADD CONSTRAINT e2e_one_time_keys_json_uniqueness UNIQUE (user_id, device_id, algorithm, key_id); + + +-- +-- Name: event_backward_extremities event_backward_extremities_event_id_room_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY event_backward_extremities + ADD CONSTRAINT event_backward_extremities_event_id_room_id_key UNIQUE (event_id, room_id); + + +-- +-- Name: event_content_hashes event_content_hashes_event_id_algorithm_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY event_content_hashes + ADD CONSTRAINT event_content_hashes_event_id_algorithm_key UNIQUE (event_id, algorithm); + + +-- +-- Name: event_destinations event_destinations_event_id_destination_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY event_destinations + ADD CONSTRAINT event_destinations_event_id_destination_key UNIQUE (event_id, destination); + + +-- +-- Name: event_edge_hashes event_edge_hashes_event_id_prev_event_id_algorithm_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY event_edge_hashes + ADD CONSTRAINT event_edge_hashes_event_id_prev_event_id_algorithm_key UNIQUE (event_id, prev_event_id, algorithm); + + +-- +-- Name: event_edges event_edges_event_id_prev_event_id_room_id_is_state_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY event_edges + ADD CONSTRAINT event_edges_event_id_prev_event_id_room_id_is_state_key UNIQUE (event_id, prev_event_id, room_id, is_state); + + +-- +-- Name: event_forward_extremities event_forward_extremities_event_id_room_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY event_forward_extremities + ADD CONSTRAINT event_forward_extremities_event_id_room_id_key UNIQUE (event_id, room_id); + + +-- +-- Name: event_push_actions event_id_user_id_profile_tag_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY event_push_actions + ADD CONSTRAINT event_id_user_id_profile_tag_uniqueness UNIQUE (room_id, event_id, user_id, profile_tag); + + +-- +-- Name: event_json event_json_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY event_json + ADD CONSTRAINT event_json_event_id_key UNIQUE (event_id); + + +-- +-- Name: event_push_summary_stream_ordering event_push_summary_stream_ordering_lock_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY event_push_summary_stream_ordering + ADD CONSTRAINT event_push_summary_stream_ordering_lock_key UNIQUE (lock); + + +-- +-- Name: event_reference_hashes event_reference_hashes_event_id_algorithm_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY event_reference_hashes + ADD CONSTRAINT event_reference_hashes_event_id_algorithm_key UNIQUE (event_id, algorithm); + + +-- +-- Name: event_reports event_reports_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY event_reports + ADD CONSTRAINT event_reports_pkey PRIMARY KEY (id); + + +-- +-- Name: event_signatures event_signatures_event_id_signature_name_key_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY event_signatures + ADD CONSTRAINT event_signatures_event_id_signature_name_key_id_key UNIQUE (event_id, signature_name, key_id); + + +-- +-- Name: event_to_state_groups event_to_state_groups_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY event_to_state_groups + ADD CONSTRAINT event_to_state_groups_event_id_key UNIQUE (event_id); + + +-- +-- Name: events events_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY events + ADD CONSTRAINT events_event_id_key UNIQUE (event_id); + + +-- +-- Name: events events_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY events + ADD CONSTRAINT events_pkey PRIMARY KEY (stream_ordering); + + +-- +-- Name: ex_outlier_stream ex_outlier_stream_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY ex_outlier_stream + ADD CONSTRAINT ex_outlier_stream_pkey PRIMARY KEY (event_stream_ordering); + + +-- +-- Name: feedback feedback_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY feedback + ADD CONSTRAINT feedback_event_id_key UNIQUE (event_id); + + +-- +-- Name: group_roles group_roles_group_id_role_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY group_roles + ADD CONSTRAINT group_roles_group_id_role_id_key UNIQUE (group_id, role_id); + + +-- +-- Name: group_room_categories group_room_categories_group_id_category_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY group_room_categories + ADD CONSTRAINT group_room_categories_group_id_category_id_key UNIQUE (group_id, category_id); + + +-- +-- Name: group_summary_roles group_summary_roles_group_id_role_id_role_order_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY group_summary_roles + ADD CONSTRAINT group_summary_roles_group_id_role_id_role_order_key UNIQUE (group_id, role_id, role_order); + + +-- +-- Name: group_summary_room_categories group_summary_room_categories_group_id_category_id_cat_orde_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY group_summary_room_categories + ADD CONSTRAINT group_summary_room_categories_group_id_category_id_cat_orde_key UNIQUE (group_id, category_id, cat_order); + + +-- +-- Name: group_summary_rooms group_summary_rooms_group_id_category_id_room_id_room_order_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY group_summary_rooms + ADD CONSTRAINT group_summary_rooms_group_id_category_id_room_id_room_order_key UNIQUE (group_id, category_id, room_id, room_order); + + +-- +-- Name: guest_access guest_access_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY guest_access + ADD CONSTRAINT guest_access_event_id_key UNIQUE (event_id); + + +-- +-- Name: history_visibility history_visibility_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY history_visibility + ADD CONSTRAINT history_visibility_event_id_key UNIQUE (event_id); + + +-- +-- Name: local_media_repository local_media_repository_media_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY local_media_repository + ADD CONSTRAINT local_media_repository_media_id_key UNIQUE (media_id); + + +-- +-- Name: local_media_repository_thumbnails local_media_repository_thumbn_media_id_thumbnail_width_thum_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY local_media_repository_thumbnails + ADD CONSTRAINT local_media_repository_thumbn_media_id_thumbnail_width_thum_key UNIQUE (media_id, thumbnail_width, thumbnail_height, thumbnail_type); + + +-- +-- Name: user_threepids medium_address; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY user_threepids + ADD CONSTRAINT medium_address UNIQUE (medium, address); + + +-- +-- Name: open_id_tokens open_id_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY open_id_tokens + ADD CONSTRAINT open_id_tokens_pkey PRIMARY KEY (token); + + +-- +-- Name: presence_allow_inbound presence_allow_inbound_observed_user_id_observer_user_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY presence_allow_inbound + ADD CONSTRAINT presence_allow_inbound_observed_user_id_observer_user_id_key UNIQUE (observed_user_id, observer_user_id); + + +-- +-- Name: presence_list presence_list_user_id_observed_user_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY presence_list + ADD CONSTRAINT presence_list_user_id_observed_user_id_key UNIQUE (user_id, observed_user_id); + + +-- +-- Name: presence presence_user_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY presence + ADD CONSTRAINT presence_user_id_key UNIQUE (user_id); + + +-- +-- Name: account_data_max_stream_id private_user_data_max_stream_id_lock_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY account_data_max_stream_id + ADD CONSTRAINT private_user_data_max_stream_id_lock_key UNIQUE (lock); + + +-- +-- Name: profiles profiles_user_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY profiles + ADD CONSTRAINT profiles_user_id_key UNIQUE (user_id); + + +-- +-- Name: push_rules_enable push_rules_enable_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY push_rules_enable + ADD CONSTRAINT push_rules_enable_pkey PRIMARY KEY (id); + + +-- +-- Name: push_rules_enable push_rules_enable_user_name_rule_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY push_rules_enable + ADD CONSTRAINT push_rules_enable_user_name_rule_id_key UNIQUE (user_name, rule_id); + + +-- +-- Name: push_rules push_rules_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY push_rules + ADD CONSTRAINT push_rules_pkey PRIMARY KEY (id); + + +-- +-- Name: push_rules push_rules_user_name_rule_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY push_rules + ADD CONSTRAINT push_rules_user_name_rule_id_key UNIQUE (user_name, rule_id); + + +-- +-- Name: pusher_throttle pusher_throttle_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY pusher_throttle + ADD CONSTRAINT pusher_throttle_pkey PRIMARY KEY (pusher, room_id); + + +-- +-- Name: pushers pushers2_app_id_pushkey_user_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY pushers + ADD CONSTRAINT pushers2_app_id_pushkey_user_name_key UNIQUE (app_id, pushkey, user_name); + + +-- +-- Name: pushers pushers2_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY pushers + ADD CONSTRAINT pushers2_pkey PRIMARY KEY (id); + + +-- +-- Name: receipts_graph receipts_graph_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY receipts_graph + ADD CONSTRAINT receipts_graph_uniqueness UNIQUE (room_id, receipt_type, user_id); + + +-- +-- Name: receipts_linearized receipts_linearized_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY receipts_linearized + ADD CONSTRAINT receipts_linearized_uniqueness UNIQUE (room_id, receipt_type, user_id); + + +-- +-- Name: received_transactions received_transactions_transaction_id_origin_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY received_transactions + ADD CONSTRAINT received_transactions_transaction_id_origin_key UNIQUE (transaction_id, origin); + + +-- +-- Name: redactions redactions_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY redactions + ADD CONSTRAINT redactions_event_id_key UNIQUE (event_id); + + +-- +-- Name: rejections rejections_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY rejections + ADD CONSTRAINT rejections_event_id_key UNIQUE (event_id); + + +-- +-- Name: remote_media_cache remote_media_cache_media_origin_media_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY remote_media_cache + ADD CONSTRAINT remote_media_cache_media_origin_media_id_key UNIQUE (media_origin, media_id); + + +-- +-- Name: remote_media_cache_thumbnails remote_media_cache_thumbnails_media_origin_media_id_thumbna_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY remote_media_cache_thumbnails + ADD CONSTRAINT remote_media_cache_thumbnails_media_origin_media_id_thumbna_key UNIQUE (media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type); + + +-- +-- Name: room_account_data room_account_data_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY room_account_data + ADD CONSTRAINT room_account_data_uniqueness UNIQUE (user_id, room_id, account_data_type); + + +-- +-- Name: room_aliases room_aliases_room_alias_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY room_aliases + ADD CONSTRAINT room_aliases_room_alias_key UNIQUE (room_alias); + + +-- +-- Name: room_depth room_depth_room_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY room_depth + ADD CONSTRAINT room_depth_room_id_key UNIQUE (room_id); + + +-- +-- Name: room_hosts room_hosts_room_id_host_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY room_hosts + ADD CONSTRAINT room_hosts_room_id_host_key UNIQUE (room_id, host); + + +-- +-- Name: room_memberships room_memberships_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY room_memberships + ADD CONSTRAINT room_memberships_event_id_key UNIQUE (event_id); + + +-- +-- Name: room_names room_names_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY room_names + ADD CONSTRAINT room_names_event_id_key UNIQUE (event_id); + + +-- +-- Name: room_tags_revisions room_tag_revisions_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY room_tags_revisions + ADD CONSTRAINT room_tag_revisions_uniqueness UNIQUE (user_id, room_id); + + +-- +-- Name: room_tags room_tag_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY room_tags + ADD CONSTRAINT room_tag_uniqueness UNIQUE (user_id, room_id, tag); + + +-- +-- Name: rooms rooms_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY rooms + ADD CONSTRAINT rooms_pkey PRIMARY KEY (room_id); + + +-- +-- Name: server_keys_json server_keys_json_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY server_keys_json + ADD CONSTRAINT server_keys_json_uniqueness UNIQUE (server_name, key_id, from_server); + + +-- +-- Name: server_signature_keys server_signature_keys_server_name_key_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY server_signature_keys + ADD CONSTRAINT server_signature_keys_server_name_key_id_key UNIQUE (server_name, key_id); + + +-- +-- Name: server_tls_certificates server_tls_certificates_server_name_fingerprint_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY server_tls_certificates + ADD CONSTRAINT server_tls_certificates_server_name_fingerprint_key UNIQUE (server_name, fingerprint); + + +-- +-- Name: state_events state_events_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY state_events + ADD CONSTRAINT state_events_event_id_key UNIQUE (event_id); + + +-- +-- Name: state_forward_extremities state_forward_extremities_event_id_room_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY state_forward_extremities + ADD CONSTRAINT state_forward_extremities_event_id_room_id_key UNIQUE (event_id, room_id); + + +-- +-- Name: state_groups state_groups_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY state_groups + ADD CONSTRAINT state_groups_pkey PRIMARY KEY (id); + + +-- +-- Name: topics topics_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY topics + ADD CONSTRAINT topics_event_id_key UNIQUE (event_id); + + +-- +-- Name: transaction_id_to_pdu transaction_id_to_pdu_transaction_id_destination_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY transaction_id_to_pdu + ADD CONSTRAINT transaction_id_to_pdu_transaction_id_destination_key UNIQUE (transaction_id, destination); + + +-- +-- Name: user_directory_stream_pos user_directory_stream_pos_lock_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY user_directory_stream_pos + ADD CONSTRAINT user_directory_stream_pos_lock_key UNIQUE (lock); + + +-- +-- Name: users users_name_key; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY users + ADD CONSTRAINT users_name_key UNIQUE (name); + + +-- +-- Name: access_tokens_device_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX access_tokens_device_id ON access_tokens USING btree (user_id, device_id); + + +-- +-- Name: account_data_stream_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX account_data_stream_id ON account_data USING btree (user_id, stream_id); + + +-- +-- Name: application_services_txns_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX application_services_txns_id ON application_services_txns USING btree (as_id); + + +-- +-- Name: appservice_room_list_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX appservice_room_list_idx ON appservice_room_list USING btree (appservice_id, network_id, room_id); + + +-- +-- Name: blocked_rooms_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX blocked_rooms_idx ON blocked_rooms USING btree (room_id); + + +-- +-- Name: cache_invalidation_stream_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX cache_invalidation_stream_id ON cache_invalidation_stream USING btree (stream_id); + + +-- +-- Name: current_state_delta_stream_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX current_state_delta_stream_idx ON current_state_delta_stream USING btree (stream_id); + + +-- +-- Name: current_state_events_member_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX current_state_events_member_index ON current_state_events USING btree (state_key) WHERE (type = 'm.room.member'::text); + + +-- +-- Name: deleted_pushers_stream_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX deleted_pushers_stream_id ON deleted_pushers USING btree (stream_id); + + +-- +-- Name: device_federation_inbox_sender_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX device_federation_inbox_sender_id ON device_federation_inbox USING btree (origin, message_id); + + +-- +-- Name: device_federation_outbox_destination_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX device_federation_outbox_destination_id ON device_federation_outbox USING btree (destination, stream_id); + + +-- +-- Name: device_federation_outbox_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX device_federation_outbox_id ON device_federation_outbox USING btree (stream_id); + + +-- +-- Name: device_inbox_stream_id_user_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX device_inbox_stream_id_user_id ON device_inbox USING btree (stream_id, user_id); + + +-- +-- Name: device_inbox_user_stream_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX device_inbox_user_stream_id ON device_inbox USING btree (user_id, device_id, stream_id); + + +-- +-- Name: device_lists_outbound_last_success_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX device_lists_outbound_last_success_idx ON device_lists_outbound_last_success USING btree (destination, user_id, stream_id); + + +-- +-- Name: device_lists_outbound_pokes_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX device_lists_outbound_pokes_id ON device_lists_outbound_pokes USING btree (destination, stream_id); + + +-- +-- Name: device_lists_outbound_pokes_stream; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX device_lists_outbound_pokes_stream ON device_lists_outbound_pokes USING btree (stream_id); + + +-- +-- Name: device_lists_outbound_pokes_user; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX device_lists_outbound_pokes_user ON device_lists_outbound_pokes USING btree (destination, user_id); + + +-- +-- Name: device_lists_remote_cache_unique_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX device_lists_remote_cache_unique_id ON device_lists_remote_cache USING btree (user_id, device_id); + + +-- +-- Name: device_lists_remote_extremeties_unique_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX device_lists_remote_extremeties_unique_idx ON device_lists_remote_extremeties USING btree (user_id); + + +-- +-- Name: device_lists_stream_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX device_lists_stream_id ON device_lists_stream USING btree (stream_id, user_id); + + +-- +-- Name: device_lists_stream_user_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX device_lists_stream_user_id ON device_lists_stream USING btree (user_id, device_id); + + +-- +-- Name: e2e_room_keys_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX e2e_room_keys_idx ON e2e_room_keys USING btree (user_id, room_id, session_id); + + +-- +-- Name: e2e_room_keys_versions_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions USING btree (user_id, version); + + +-- +-- Name: erased_users_user; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX erased_users_user ON erased_users USING btree (user_id); + + +-- +-- Name: ev_b_extrem_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ev_b_extrem_id ON event_backward_extremities USING btree (event_id); + + +-- +-- Name: ev_b_extrem_room; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ev_b_extrem_room ON event_backward_extremities USING btree (room_id); + + +-- +-- Name: ev_edges_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ev_edges_id ON event_edges USING btree (event_id); + + +-- +-- Name: ev_edges_prev_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ev_edges_prev_id ON event_edges USING btree (prev_event_id); + + +-- +-- Name: ev_extrem_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ev_extrem_id ON event_forward_extremities USING btree (event_id); + + +-- +-- Name: ev_extrem_room; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ev_extrem_room ON event_forward_extremities USING btree (room_id); + + +-- +-- Name: evauth_edges_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX evauth_edges_id ON event_auth USING btree (event_id); + + +-- +-- Name: event_contains_url_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX event_contains_url_index ON events USING btree (room_id, topological_ordering, stream_ordering) WHERE ((contains_url = true) AND (outlier = false)); + + +-- +-- Name: event_json_room_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX event_json_room_id ON event_json USING btree (room_id); + + +-- +-- Name: event_push_actions_highlights_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX event_push_actions_highlights_index ON event_push_actions USING btree (user_id, room_id, topological_ordering, stream_ordering) WHERE (highlight = 1); + + +-- +-- Name: event_push_actions_rm_tokens; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX event_push_actions_rm_tokens ON event_push_actions USING btree (user_id, room_id, topological_ordering, stream_ordering); + + +-- +-- Name: event_push_actions_room_id_user_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX event_push_actions_room_id_user_id ON event_push_actions USING btree (room_id, user_id); + + +-- +-- Name: event_push_actions_staging_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX event_push_actions_staging_id ON event_push_actions_staging USING btree (event_id); + + +-- +-- Name: event_push_actions_stream_ordering; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX event_push_actions_stream_ordering ON event_push_actions USING btree (stream_ordering, user_id); + + +-- +-- Name: event_push_actions_u_highlight; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX event_push_actions_u_highlight ON event_push_actions USING btree (user_id, stream_ordering); + + +-- +-- Name: event_push_summary_user_rm; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX event_push_summary_user_rm ON event_push_summary USING btree (user_id, room_id); + + +-- +-- Name: event_reference_hashes_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX event_reference_hashes_id ON event_reference_hashes USING btree (event_id); + + +-- +-- Name: event_search_ev_ridx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX event_search_ev_ridx ON event_search USING btree (room_id); + + +-- +-- Name: event_search_event_id_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX event_search_event_id_idx ON event_search USING btree (event_id); + + +-- +-- Name: event_search_fts_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX event_search_fts_idx ON event_search USING gin (vector); + + +-- +-- Name: event_to_state_groups_sg_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX event_to_state_groups_sg_index ON event_to_state_groups USING btree (state_group); + + +-- +-- Name: events_order_room; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX events_order_room ON events USING btree (room_id, topological_ordering, stream_ordering); + + +-- +-- Name: events_room_stream; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX events_room_stream ON events USING btree (room_id, stream_ordering); + + +-- +-- Name: events_ts; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX events_ts ON events USING btree (origin_server_ts, stream_ordering); + + +-- +-- Name: group_attestations_remote_g_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX group_attestations_remote_g_idx ON group_attestations_remote USING btree (group_id, user_id); + + +-- +-- Name: group_attestations_remote_u_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX group_attestations_remote_u_idx ON group_attestations_remote USING btree (user_id); + + +-- +-- Name: group_attestations_remote_v_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX group_attestations_remote_v_idx ON group_attestations_remote USING btree (valid_until_ms); + + +-- +-- Name: group_attestations_renewals_g_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX group_attestations_renewals_g_idx ON group_attestations_renewals USING btree (group_id, user_id); + + +-- +-- Name: group_attestations_renewals_u_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX group_attestations_renewals_u_idx ON group_attestations_renewals USING btree (user_id); + + +-- +-- Name: group_attestations_renewals_v_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX group_attestations_renewals_v_idx ON group_attestations_renewals USING btree (valid_until_ms); + + +-- +-- Name: group_invites_g_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX group_invites_g_idx ON group_invites USING btree (group_id, user_id); + + +-- +-- Name: group_invites_u_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX group_invites_u_idx ON group_invites USING btree (user_id); + + +-- +-- Name: group_rooms_g_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX group_rooms_g_idx ON group_rooms USING btree (group_id, room_id); + + +-- +-- Name: group_rooms_r_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX group_rooms_r_idx ON group_rooms USING btree (room_id); + + +-- +-- Name: group_summary_rooms_g_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX group_summary_rooms_g_idx ON group_summary_rooms USING btree (group_id, room_id, category_id); + + +-- +-- Name: group_summary_users_g_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX group_summary_users_g_idx ON group_summary_users USING btree (group_id); + + +-- +-- Name: group_users_g_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX group_users_g_idx ON group_users USING btree (group_id, user_id); + + +-- +-- Name: group_users_u_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX group_users_u_idx ON group_users USING btree (user_id); + + +-- +-- Name: groups_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX groups_idx ON groups USING btree (group_id); + + +-- +-- Name: local_group_membership_g_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX local_group_membership_g_idx ON local_group_membership USING btree (group_id); + + +-- +-- Name: local_group_membership_u_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX local_group_membership_u_idx ON local_group_membership USING btree (user_id, group_id); + + +-- +-- Name: local_invites_for_user_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX local_invites_for_user_idx ON local_invites USING btree (invitee, locally_rejected, replaced_by, room_id); + + +-- +-- Name: local_invites_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX local_invites_id ON local_invites USING btree (stream_id); + + +-- +-- Name: local_media_repository_thumbnails_media_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails USING btree (media_id); + + +-- +-- Name: local_media_repository_url_cache_by_url_download_ts; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX local_media_repository_url_cache_by_url_download_ts ON local_media_repository_url_cache USING btree (url, download_ts); + + +-- +-- Name: local_media_repository_url_cache_expires_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX local_media_repository_url_cache_expires_idx ON local_media_repository_url_cache USING btree (expires_ts); + + +-- +-- Name: local_media_repository_url_cache_media_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX local_media_repository_url_cache_media_idx ON local_media_repository_url_cache USING btree (media_id); + + +-- +-- Name: local_media_repository_url_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX local_media_repository_url_idx ON local_media_repository USING btree (created_ts) WHERE (url_cache IS NOT NULL); + + +-- +-- Name: monthly_active_users_time_stamp; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX monthly_active_users_time_stamp ON monthly_active_users USING btree ("timestamp"); + + +-- +-- Name: monthly_active_users_users; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX monthly_active_users_users ON monthly_active_users USING btree (user_id); + + +-- +-- Name: open_id_tokens_ts_valid_until_ms; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX open_id_tokens_ts_valid_until_ms ON open_id_tokens USING btree (ts_valid_until_ms); + + +-- +-- Name: presence_list_user_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX presence_list_user_id ON presence_list USING btree (user_id); + + +-- +-- Name: presence_stream_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX presence_stream_id ON presence_stream USING btree (stream_id, user_id); + + +-- +-- Name: presence_stream_user_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX presence_stream_user_id ON presence_stream USING btree (user_id); + + +-- +-- Name: public_room_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX public_room_index ON rooms USING btree (is_public); + + +-- +-- Name: public_room_list_stream_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX public_room_list_stream_idx ON public_room_list_stream USING btree (stream_id); + + +-- +-- Name: public_room_list_stream_rm_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX public_room_list_stream_rm_idx ON public_room_list_stream USING btree (room_id, stream_id); + + +-- +-- Name: push_rules_enable_user_name; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX push_rules_enable_user_name ON push_rules_enable USING btree (user_name); + + +-- +-- Name: push_rules_stream_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX push_rules_stream_id ON push_rules_stream USING btree (stream_id); + + +-- +-- Name: push_rules_stream_user_stream_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX push_rules_stream_user_stream_id ON push_rules_stream USING btree (user_id, stream_id); + + +-- +-- Name: push_rules_user_name; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX push_rules_user_name ON push_rules USING btree (user_name); + + +-- +-- Name: ratelimit_override_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX ratelimit_override_idx ON ratelimit_override USING btree (user_id); + + +-- +-- Name: receipts_linearized_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX receipts_linearized_id ON receipts_linearized USING btree (stream_id); + + +-- +-- Name: receipts_linearized_room_stream; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX receipts_linearized_room_stream ON receipts_linearized USING btree (room_id, stream_id); + + +-- +-- Name: receipts_linearized_user; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX receipts_linearized_user ON receipts_linearized USING btree (user_id); + + +-- +-- Name: received_transactions_ts; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX received_transactions_ts ON received_transactions USING btree (ts); + + +-- +-- Name: redactions_redacts; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX redactions_redacts ON redactions USING btree (redacts); + + +-- +-- Name: remote_profile_cache_time; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX remote_profile_cache_time ON remote_profile_cache USING btree (last_check); + + +-- +-- Name: remote_profile_cache_user_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX remote_profile_cache_user_id ON remote_profile_cache USING btree (user_id); + + +-- +-- Name: room_account_data_stream_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX room_account_data_stream_id ON room_account_data USING btree (user_id, stream_id); + + +-- +-- Name: room_alias_servers_alias; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX room_alias_servers_alias ON room_alias_servers USING btree (room_alias); + + +-- +-- Name: room_aliases_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX room_aliases_id ON room_aliases USING btree (room_id); + + +-- +-- Name: room_depth_room; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX room_depth_room ON room_depth USING btree (room_id); + + +-- +-- Name: room_memberships_room_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX room_memberships_room_id ON room_memberships USING btree (room_id); + + +-- +-- Name: room_memberships_user_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX room_memberships_user_id ON room_memberships USING btree (user_id); + + +-- +-- Name: room_names_room_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX room_names_room_id ON room_names USING btree (room_id); + + +-- +-- Name: st_extrem_keys; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX st_extrem_keys ON state_forward_extremities USING btree (room_id, type, state_key); + + +-- +-- Name: state_group_edges_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX state_group_edges_idx ON state_group_edges USING btree (state_group); + + +-- +-- Name: state_group_edges_prev_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX state_group_edges_prev_idx ON state_group_edges USING btree (prev_state_group); + + +-- +-- Name: state_groups_state_type_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX state_groups_state_type_idx ON state_groups_state USING btree (state_group, type, state_key); + + +-- +-- Name: stream_ordering_to_exterm_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX stream_ordering_to_exterm_idx ON stream_ordering_to_exterm USING btree (stream_ordering); + + +-- +-- Name: stream_ordering_to_exterm_rm_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX stream_ordering_to_exterm_rm_idx ON stream_ordering_to_exterm USING btree (room_id, stream_ordering); + + +-- +-- Name: threepid_guest_access_tokens_index; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX threepid_guest_access_tokens_index ON threepid_guest_access_tokens USING btree (medium, address); + + +-- +-- Name: topics_room_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX topics_room_id ON topics USING btree (room_id); + + +-- +-- Name: transaction_id_to_pdu_dest; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu USING btree (destination); + + +-- +-- Name: user_daily_visits_ts_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX user_daily_visits_ts_idx ON user_daily_visits USING btree ("timestamp"); + + +-- +-- Name: user_daily_visits_uts_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX user_daily_visits_uts_idx ON user_daily_visits USING btree (user_id, "timestamp"); + + +-- +-- Name: user_directory_room_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX user_directory_room_idx ON user_directory USING btree (room_id); + + +-- +-- Name: user_directory_search_fts_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX user_directory_search_fts_idx ON user_directory_search USING gin (vector); + + +-- +-- Name: user_directory_search_user_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX user_directory_search_user_idx ON user_directory_search USING btree (user_id); + + +-- +-- Name: user_directory_user_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX user_directory_user_idx ON user_directory USING btree (user_id); + + +-- +-- Name: user_filters_by_user_id_filter_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX user_filters_by_user_id_filter_id ON user_filters USING btree (user_id, filter_id); + + +-- +-- Name: user_ips_device_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX user_ips_device_id ON user_ips USING btree (user_id, device_id, last_seen); + + +-- +-- Name: user_ips_device_unique_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX user_ips_device_unique_id ON user_ips USING btree (user_id, access_token, ip, user_agent, device_id); + + +-- +-- Name: user_ips_last_seen; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX user_ips_last_seen ON user_ips USING btree (user_id, last_seen); + + +-- +-- Name: user_ips_last_seen_only; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX user_ips_last_seen_only ON user_ips USING btree (last_seen); + + +-- +-- Name: user_ips_user_ip; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX user_ips_user_ip ON user_ips USING btree (user_id, access_token, ip); + + +-- +-- Name: user_threepids_medium_address; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX user_threepids_medium_address ON user_threepids USING btree (medium, address); + + +-- +-- Name: user_threepids_user_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX user_threepids_user_id ON user_threepids USING btree (user_id); + + +-- +-- Name: users_creation_ts; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX users_creation_ts ON users USING btree (creation_ts); + + +-- +-- Name: users_in_public_rooms_room_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX users_in_public_rooms_room_idx ON users_in_public_rooms USING btree (room_id); + + +-- +-- Name: users_in_public_rooms_user_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX users_in_public_rooms_user_idx ON users_in_public_rooms USING btree (user_id); + + +-- +-- Name: users_who_share_rooms_o_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX users_who_share_rooms_o_idx ON users_who_share_rooms USING btree (other_user_id); + + +-- +-- Name: users_who_share_rooms_r_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX users_who_share_rooms_r_idx ON users_who_share_rooms USING btree (room_id); + + +-- +-- Name: users_who_share_rooms_u_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX users_who_share_rooms_u_idx ON users_who_share_rooms USING btree (user_id, other_user_id); + + +-- +-- Name: application_services_regex application_services_regex_as_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY application_services_regex + ADD CONSTRAINT application_services_regex_as_id_fkey FOREIGN KEY (as_id) REFERENCES application_services(id); + + +-- +-- PostgreSQL database dump complete +-- + diff --git a/synapse/storage/schema/full_schemas/54/full.sql.sqlite b/synapse/storage/schema/full_schemas/54/full.sql.sqlite new file mode 100644 index 000000000000..60b8c6fe7629 --- /dev/null +++ b/synapse/storage/schema/full_schemas/54/full.sql.sqlite @@ -0,0 +1,1203 @@ +CREATE TABLE application_services( + id BIGINT PRIMARY KEY, + url TEXT, + token TEXT, + hs_token TEXT, + sender TEXT, + UNIQUE(token) +); +CREATE TABLE application_services_regex( + id BIGINT PRIMARY KEY, + as_id BIGINT NOT NULL, + namespace INTEGER, + regex TEXT, + FOREIGN KEY(as_id) REFERENCES application_services(id) +); +CREATE TABLE application_services_state( + as_id TEXT PRIMARY KEY, + state VARCHAR(5), + last_txn INTEGER +); +CREATE TABLE application_services_txns( + as_id TEXT NOT NULL, + txn_id INTEGER NOT NULL, + event_ids TEXT NOT NULL, + UNIQUE(as_id, txn_id) +); +CREATE INDEX application_services_txns_id ON application_services_txns(as_id); +CREATE TABLE presence( + user_id TEXT NOT NULL, + state VARCHAR(20), + status_msg TEXT, + mtime BIGINT, + UNIQUE(user_id) +); +CREATE TABLE presence_allow_inbound( + observed_user_id TEXT NOT NULL, + observer_user_id TEXT NOT NULL, + UNIQUE(observed_user_id, observer_user_id) +); +CREATE TABLE presence_list( + user_id TEXT NOT NULL, + observed_user_id TEXT NOT NULL, + accepted BOOLEAN NOT NULL, + UNIQUE(user_id, observed_user_id) +); +CREATE INDEX presence_list_user_id ON presence_list(user_id); +CREATE TABLE users( + name TEXT, + password_hash TEXT, + creation_ts BIGINT, + admin SMALLINT DEFAULT 0 NOT NULL, + upgrade_ts BIGINT, + is_guest SMALLINT DEFAULT 0 NOT NULL, + appservice_id TEXT, + consent_version TEXT, + consent_server_notice_sent TEXT, + user_type TEXT DEFAULT NULL, + UNIQUE(name) +); +CREATE TABLE access_tokens( + id BIGINT PRIMARY KEY, + user_id TEXT NOT NULL, + device_id TEXT, + token TEXT NOT NULL, + last_used BIGINT, + UNIQUE(token) +); +CREATE TABLE user_ips( + user_id TEXT NOT NULL, + access_token TEXT NOT NULL, + device_id TEXT, + ip TEXT NOT NULL, + user_agent TEXT NOT NULL, + last_seen BIGINT NOT NULL +); +CREATE INDEX user_ips_user_ip ON user_ips(user_id, access_token, ip); +CREATE TABLE profiles( + user_id TEXT NOT NULL, + displayname TEXT, + avatar_url TEXT, + UNIQUE(user_id) +); +CREATE TABLE received_transactions( + transaction_id TEXT, + origin TEXT, + ts BIGINT, + response_code INTEGER, + response_json bytea, + has_been_referenced smallint default 0, + UNIQUE(transaction_id, origin) +); +CREATE TABLE transaction_id_to_pdu( + transaction_id INTEGER, + destination TEXT, + pdu_id TEXT, + pdu_origin TEXT, + UNIQUE(transaction_id, destination) +); +CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); +CREATE TABLE destinations( + destination TEXT PRIMARY KEY, + retry_last_ts BIGINT, + retry_interval INTEGER +); +CREATE TABLE events( + stream_ordering INTEGER PRIMARY KEY, + topological_ordering BIGINT NOT NULL, + event_id TEXT NOT NULL, + type TEXT NOT NULL, + room_id TEXT NOT NULL, + content TEXT, + unrecognized_keys TEXT, + processed BOOL NOT NULL, + outlier BOOL NOT NULL, + depth BIGINT DEFAULT 0 NOT NULL, + origin_server_ts BIGINT, + received_ts BIGINT, + sender TEXT, + contains_url BOOLEAN, + UNIQUE(event_id) +); +CREATE INDEX events_order_room ON events( + room_id, + topological_ordering, + stream_ordering +); +CREATE TABLE event_json( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + internal_metadata TEXT NOT NULL, + json TEXT NOT NULL, + UNIQUE(event_id) +); +CREATE INDEX event_json_room_id ON event_json(room_id); +CREATE TABLE state_events( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + prev_state TEXT, + UNIQUE(event_id) +); +CREATE TABLE current_state_events( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + UNIQUE(event_id), + UNIQUE(room_id, type, state_key) +); +CREATE TABLE room_memberships( + event_id TEXT NOT NULL, + user_id TEXT NOT NULL, + sender TEXT NOT NULL, + room_id TEXT NOT NULL, + membership TEXT NOT NULL, + forgotten INTEGER DEFAULT 0, + display_name TEXT, + avatar_url TEXT, + UNIQUE(event_id) +); +CREATE INDEX room_memberships_room_id ON room_memberships(room_id); +CREATE INDEX room_memberships_user_id ON room_memberships(user_id); +CREATE TABLE feedback( + event_id TEXT NOT NULL, + feedback_type TEXT, + target_event_id TEXT, + sender TEXT, + room_id TEXT, + UNIQUE(event_id) +); +CREATE TABLE topics( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + topic TEXT NOT NULL, + UNIQUE(event_id) +); +CREATE INDEX topics_room_id ON topics(room_id); +CREATE TABLE room_names( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + name TEXT NOT NULL, + UNIQUE(event_id) +); +CREATE INDEX room_names_room_id ON room_names(room_id); +CREATE TABLE rooms( + room_id TEXT PRIMARY KEY NOT NULL, + is_public BOOL, + creator TEXT +); +CREATE TABLE room_hosts( + room_id TEXT NOT NULL, + host TEXT NOT NULL, + UNIQUE(room_id, host) +); +CREATE TABLE server_tls_certificates( + server_name TEXT, + fingerprint TEXT, + from_server TEXT, + ts_added_ms BIGINT, + tls_certificate bytea, + UNIQUE(server_name, fingerprint) +); +CREATE TABLE server_signature_keys( + server_name TEXT, + key_id TEXT, + from_server TEXT, + ts_added_ms BIGINT, + verify_key bytea, + UNIQUE(server_name, key_id) +); +CREATE TABLE rejections( + event_id TEXT NOT NULL, + reason TEXT NOT NULL, + last_check TEXT NOT NULL, + UNIQUE(event_id) +); +CREATE TABLE push_rules( + id BIGINT PRIMARY KEY, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, + priority_class SMALLINT NOT NULL, + priority INTEGER NOT NULL DEFAULT 0, + conditions TEXT NOT NULL, + actions TEXT NOT NULL, + UNIQUE(user_name, rule_id) +); +CREATE INDEX push_rules_user_name on push_rules(user_name); +CREATE TABLE user_filters(user_id TEXT, filter_id BIGINT, filter_json bytea); +CREATE INDEX user_filters_by_user_id_filter_id ON user_filters( + user_id, + filter_id +); +CREATE TABLE push_rules_enable( + id BIGINT PRIMARY KEY, + user_name TEXT NOT NULL, + rule_id TEXT NOT NULL, + enabled SMALLINT, + UNIQUE(user_name, rule_id) +); +CREATE INDEX push_rules_enable_user_name on push_rules_enable(user_name); +CREATE TABLE event_forward_extremities( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + UNIQUE(event_id, room_id) +); +CREATE INDEX ev_extrem_room ON event_forward_extremities(room_id); +CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id); +CREATE TABLE event_backward_extremities( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + UNIQUE(event_id, room_id) +); +CREATE INDEX ev_b_extrem_room ON event_backward_extremities(room_id); +CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id); +CREATE TABLE event_edges( + event_id TEXT NOT NULL, + prev_event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + is_state BOOL NOT NULL, + UNIQUE(event_id, prev_event_id, room_id, is_state) +); +CREATE INDEX ev_edges_id ON event_edges(event_id); +CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id); +CREATE TABLE room_depth( + room_id TEXT NOT NULL, + min_depth INTEGER NOT NULL, + UNIQUE(room_id) +); +CREATE INDEX room_depth_room ON room_depth(room_id); +CREATE TABLE event_destinations( + event_id TEXT NOT NULL, + destination TEXT NOT NULL, + delivered_ts BIGINT DEFAULT 0, + UNIQUE(event_id, destination) +); +CREATE TABLE state_forward_extremities( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + UNIQUE(event_id, room_id) +); +CREATE INDEX st_extrem_keys ON state_forward_extremities( + room_id, + type, + state_key +); +CREATE TABLE state_groups( + id BIGINT PRIMARY KEY, + room_id TEXT NOT NULL, + event_id TEXT NOT NULL +); +CREATE TABLE state_groups_state( + state_group BIGINT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + event_id TEXT NOT NULL +); +CREATE TABLE event_to_state_groups( + event_id TEXT NOT NULL, + state_group BIGINT NOT NULL, + UNIQUE(event_id) +); +CREATE TABLE local_media_repository( + media_id TEXT, + media_type TEXT, + media_length INTEGER, + created_ts BIGINT, + upload_name TEXT, + user_id TEXT, + quarantined_by TEXT, + url_cache TEXT, + last_access_ts BIGINT, + UNIQUE(media_id) +); +CREATE TABLE local_media_repository_thumbnails( + media_id TEXT, + thumbnail_width INTEGER, + thumbnail_height INTEGER, + thumbnail_type TEXT, + thumbnail_method TEXT, + thumbnail_length INTEGER, + UNIQUE(media_id, thumbnail_width, thumbnail_height, thumbnail_type) +); +CREATE INDEX local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails( + media_id +); +CREATE TABLE remote_media_cache( + media_origin TEXT, + media_id TEXT, + media_type TEXT, + created_ts BIGINT, + upload_name TEXT, + media_length INTEGER, + filesystem_id TEXT, + last_access_ts BIGINT, + quarantined_by TEXT, + UNIQUE(media_origin, media_id) +); +CREATE TABLE remote_media_cache_thumbnails( + media_origin TEXT, + media_id TEXT, + thumbnail_width INTEGER, + thumbnail_height INTEGER, + thumbnail_method TEXT, + thumbnail_type TEXT, + thumbnail_length INTEGER, + filesystem_id TEXT, + UNIQUE(media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type) +); +CREATE TABLE redactions( + event_id TEXT NOT NULL, + redacts TEXT NOT NULL, + UNIQUE(event_id) +); +CREATE INDEX redactions_redacts ON redactions(redacts); +CREATE TABLE room_aliases( + room_alias TEXT NOT NULL, + room_id TEXT NOT NULL, + creator TEXT, + UNIQUE(room_alias) +); +CREATE INDEX room_aliases_id ON room_aliases(room_id); +CREATE TABLE room_alias_servers( + room_alias TEXT NOT NULL, + server TEXT NOT NULL +); +CREATE INDEX room_alias_servers_alias ON room_alias_servers(room_alias); +CREATE TABLE event_content_hashes( + event_id TEXT, + algorithm TEXT, + hash bytea, + UNIQUE(event_id, algorithm) +); +CREATE TABLE event_reference_hashes( + event_id TEXT, + algorithm TEXT, + hash bytea, + UNIQUE(event_id, algorithm) +); +CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id); +CREATE TABLE event_signatures( + event_id TEXT, + signature_name TEXT, + key_id TEXT, + signature bytea, + UNIQUE(event_id, signature_name, key_id) +); +CREATE TABLE event_edge_hashes( + event_id TEXT, + prev_event_id TEXT, + algorithm TEXT, + hash bytea, + UNIQUE(event_id, prev_event_id, algorithm) +); +CREATE TABLE IF NOT EXISTS "server_keys_json"( + server_name TEXT NOT NULL, + key_id TEXT NOT NULL, + from_server TEXT NOT NULL, + ts_added_ms BIGINT NOT NULL, + ts_valid_until_ms BIGINT NOT NULL, + key_json bytea NOT NULL, + CONSTRAINT server_keys_json_uniqueness UNIQUE(server_name, key_id, from_server) +); +CREATE TABLE e2e_device_keys_json( + user_id TEXT NOT NULL, + device_id TEXT NOT NULL, + ts_added_ms BIGINT NOT NULL, + key_json TEXT NOT NULL, + CONSTRAINT e2e_device_keys_json_uniqueness UNIQUE(user_id, device_id) +); +CREATE TABLE e2e_one_time_keys_json( + user_id TEXT NOT NULL, + device_id TEXT NOT NULL, + algorithm TEXT NOT NULL, + key_id TEXT NOT NULL, + ts_added_ms BIGINT NOT NULL, + key_json TEXT NOT NULL, + CONSTRAINT e2e_one_time_keys_json_uniqueness UNIQUE(user_id, device_id, algorithm, key_id) +); +CREATE TABLE receipts_graph( + room_id TEXT NOT NULL, + receipt_type TEXT NOT NULL, + user_id TEXT NOT NULL, + event_ids TEXT NOT NULL, + data TEXT NOT NULL, + CONSTRAINT receipts_graph_uniqueness UNIQUE(room_id, receipt_type, user_id) +); +CREATE TABLE receipts_linearized( + stream_id BIGINT NOT NULL, + room_id TEXT NOT NULL, + receipt_type TEXT NOT NULL, + user_id TEXT NOT NULL, + event_id TEXT NOT NULL, + data TEXT NOT NULL, + CONSTRAINT receipts_linearized_uniqueness UNIQUE(room_id, receipt_type, user_id) +); +CREATE INDEX receipts_linearized_id ON receipts_linearized(stream_id); +CREATE INDEX receipts_linearized_room_stream ON receipts_linearized( + room_id, + stream_id +); +CREATE TABLE IF NOT EXISTS "user_threepids"( + user_id TEXT NOT NULL, + medium TEXT NOT NULL, + address TEXT NOT NULL, + validated_at BIGINT NOT NULL, + added_at BIGINT NOT NULL, + CONSTRAINT medium_address UNIQUE(medium, address) +); +CREATE INDEX user_threepids_user_id ON user_threepids(user_id); +CREATE TABLE stats_reporting( + reported_stream_token INTEGER, + reported_time BIGINT +); +CREATE TABLE background_updates( + update_name TEXT NOT NULL, + progress_json TEXT NOT NULL, + depends_on TEXT, + CONSTRAINT background_updates_uniqueness UNIQUE(update_name) +); +CREATE VIRTUAL TABLE event_search USING fts4( + event_id, + room_id, + sender, + key, + value +) +/* event_search( + event_id, + room_id, + sender, + "key", + value +) */; +CREATE TABLE IF NOT EXISTS 'event_search_content'( + docid INTEGER PRIMARY KEY, + 'c0event_id', + 'c1room_id', + 'c2sender', + 'c3key', + 'c4value' +); +CREATE TABLE IF NOT EXISTS 'event_search_segments'(blockid INTEGER PRIMARY KEY, block BLOB); +CREATE TABLE IF NOT EXISTS 'event_search_segdir'( + level INTEGER, + idx INTEGER, + start_block INTEGER, + leaves_end_block INTEGER, + end_block INTEGER, + root BLOB, + PRIMARY KEY(level, idx) +); +CREATE TABLE IF NOT EXISTS 'event_search_docsize'(docid INTEGER PRIMARY KEY, size BLOB); +CREATE TABLE IF NOT EXISTS 'event_search_stat'(id INTEGER PRIMARY KEY, value BLOB); +CREATE TABLE guest_access( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + guest_access TEXT NOT NULL, + UNIQUE(event_id) +); +CREATE TABLE history_visibility( + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + history_visibility TEXT NOT NULL, + UNIQUE(event_id) +); +CREATE TABLE room_tags( + user_id TEXT NOT NULL, + room_id TEXT NOT NULL, + tag TEXT NOT NULL, + content TEXT NOT NULL, + CONSTRAINT room_tag_uniqueness UNIQUE(user_id, room_id, tag) +); +CREATE TABLE room_tags_revisions( + user_id TEXT NOT NULL, + room_id TEXT NOT NULL, + stream_id BIGINT NOT NULL, + CONSTRAINT room_tag_revisions_uniqueness UNIQUE(user_id, room_id) +); +CREATE TABLE IF NOT EXISTS "account_data_max_stream_id"( + Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, + stream_id BIGINT NOT NULL, + CHECK(Lock='X') +); +CREATE TABLE account_data( + user_id TEXT NOT NULL, + account_data_type TEXT NOT NULL, + stream_id BIGINT NOT NULL, + content TEXT NOT NULL, + CONSTRAINT account_data_uniqueness UNIQUE(user_id, account_data_type) +); +CREATE TABLE room_account_data( + user_id TEXT NOT NULL, + room_id TEXT NOT NULL, + account_data_type TEXT NOT NULL, + stream_id BIGINT NOT NULL, + content TEXT NOT NULL, + CONSTRAINT room_account_data_uniqueness UNIQUE(user_id, room_id, account_data_type) +); +CREATE INDEX account_data_stream_id on account_data(user_id, stream_id); +CREATE INDEX room_account_data_stream_id on room_account_data( + user_id, + stream_id +); +CREATE INDEX events_ts ON events(origin_server_ts, stream_ordering); +CREATE TABLE event_push_actions( + room_id TEXT NOT NULL, + event_id TEXT NOT NULL, + user_id TEXT NOT NULL, + profile_tag VARCHAR(32), + actions TEXT NOT NULL, + topological_ordering BIGINT, + stream_ordering BIGINT, + notif SMALLINT, + highlight SMALLINT, + CONSTRAINT event_id_user_id_profile_tag_uniqueness UNIQUE(room_id, event_id, user_id, profile_tag) +); +CREATE INDEX event_push_actions_room_id_user_id on event_push_actions( + room_id, + user_id +); +CREATE INDEX events_room_stream on events(room_id, stream_ordering); +CREATE INDEX public_room_index on rooms(is_public); +CREATE INDEX receipts_linearized_user ON receipts_linearized(user_id); +CREATE INDEX event_push_actions_rm_tokens on event_push_actions( + user_id, + room_id, + topological_ordering, + stream_ordering +); +CREATE TABLE presence_stream( + stream_id BIGINT, + user_id TEXT, + state TEXT, + last_active_ts BIGINT, + last_federation_update_ts BIGINT, + last_user_sync_ts BIGINT, + status_msg TEXT, + currently_active BOOLEAN +); +CREATE INDEX presence_stream_id ON presence_stream(stream_id, user_id); +CREATE INDEX presence_stream_user_id ON presence_stream(user_id); +CREATE TABLE push_rules_stream( + stream_id BIGINT NOT NULL, + event_stream_ordering BIGINT NOT NULL, + user_id TEXT NOT NULL, + rule_id TEXT NOT NULL, + op TEXT NOT NULL, + priority_class SMALLINT, + priority INTEGER, + conditions TEXT, + actions TEXT +); +CREATE INDEX push_rules_stream_id ON push_rules_stream(stream_id); +CREATE INDEX push_rules_stream_user_stream_id on push_rules_stream( + user_id, + stream_id +); +CREATE TABLE current_state_resets( + event_stream_ordering BIGINT PRIMARY KEY NOT NULL +); +CREATE TABLE ex_outlier_stream( + event_stream_ordering BIGINT PRIMARY KEY NOT NULL, + event_id TEXT NOT NULL, + state_group BIGINT NOT NULL +); +CREATE TABLE threepid_guest_access_tokens( + medium TEXT, + address TEXT, + guest_access_token TEXT, + first_inviter TEXT +); +CREATE UNIQUE INDEX threepid_guest_access_tokens_index ON threepid_guest_access_tokens( + medium, + address +); +CREATE TABLE local_invites( + stream_id BIGINT NOT NULL, + inviter TEXT NOT NULL, + invitee TEXT NOT NULL, + event_id TEXT NOT NULL, + room_id TEXT NOT NULL, + locally_rejected TEXT, + replaced_by TEXT +); +CREATE INDEX local_invites_id ON local_invites(stream_id); +CREATE INDEX local_invites_for_user_idx ON local_invites( + invitee, + locally_rejected, + replaced_by, + room_id +); +CREATE INDEX event_push_actions_stream_ordering on event_push_actions( + stream_ordering, + user_id +); +CREATE TABLE open_id_tokens( + token TEXT NOT NULL PRIMARY KEY, + ts_valid_until_ms bigint NOT NULL, + user_id TEXT NOT NULL, + UNIQUE(token) +); +CREATE INDEX open_id_tokens_ts_valid_until_ms ON open_id_tokens( + ts_valid_until_ms +); +CREATE TABLE pusher_throttle( + pusher BIGINT NOT NULL, + room_id TEXT NOT NULL, + last_sent_ts BIGINT, + throttle_ms BIGINT, + PRIMARY KEY(pusher, room_id) +); +CREATE TABLE event_reports( + id BIGINT NOT NULL PRIMARY KEY, + received_ts BIGINT NOT NULL, + room_id TEXT NOT NULL, + event_id TEXT NOT NULL, + user_id TEXT NOT NULL, + reason TEXT, + content TEXT +); +CREATE TABLE devices( + user_id TEXT NOT NULL, + device_id TEXT NOT NULL, + display_name TEXT, + CONSTRAINT device_uniqueness UNIQUE(user_id, device_id) +); +CREATE TABLE appservice_stream_position( + Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, + stream_ordering BIGINT, + CHECK(Lock='X') +); +CREATE TABLE device_inbox( + user_id TEXT NOT NULL, + device_id TEXT NOT NULL, + stream_id BIGINT NOT NULL, + message_json TEXT NOT NULL +); +CREATE INDEX device_inbox_user_stream_id ON device_inbox( + user_id, + device_id, + stream_id +); +CREATE INDEX received_transactions_ts ON received_transactions(ts); +CREATE TABLE device_federation_outbox( + destination TEXT NOT NULL, + stream_id BIGINT NOT NULL, + queued_ts BIGINT NOT NULL, + messages_json TEXT NOT NULL +); +CREATE INDEX device_federation_outbox_destination_id ON device_federation_outbox( + destination, + stream_id +); +CREATE TABLE device_federation_inbox( + origin TEXT NOT NULL, + message_id TEXT NOT NULL, + received_ts BIGINT NOT NULL +); +CREATE INDEX device_federation_inbox_sender_id ON device_federation_inbox( + origin, + message_id +); +CREATE TABLE device_max_stream_id(stream_id BIGINT NOT NULL); +CREATE TABLE public_room_list_stream( + stream_id BIGINT NOT NULL, + room_id TEXT NOT NULL, + visibility BOOLEAN NOT NULL , + appservice_id TEXT, + network_id TEXT +); +CREATE INDEX public_room_list_stream_idx on public_room_list_stream(stream_id); +CREATE INDEX public_room_list_stream_rm_idx on public_room_list_stream( + room_id, + stream_id +); +CREATE TABLE state_group_edges( + state_group BIGINT NOT NULL, + prev_state_group BIGINT NOT NULL +); +CREATE INDEX state_group_edges_idx ON state_group_edges(state_group); +CREATE INDEX state_group_edges_prev_idx ON state_group_edges(prev_state_group); +CREATE TABLE stream_ordering_to_exterm( + stream_ordering BIGINT NOT NULL, + room_id TEXT NOT NULL, + event_id TEXT NOT NULL +); +CREATE INDEX stream_ordering_to_exterm_idx on stream_ordering_to_exterm( + stream_ordering +); +CREATE INDEX stream_ordering_to_exterm_rm_idx on stream_ordering_to_exterm( + room_id, + stream_ordering +); +CREATE TABLE IF NOT EXISTS "event_auth"( + event_id TEXT NOT NULL, + auth_id TEXT NOT NULL, + room_id TEXT NOT NULL +); +CREATE INDEX evauth_edges_id ON event_auth(event_id); +CREATE INDEX user_threepids_medium_address on user_threepids(medium, address); +CREATE TABLE appservice_room_list( + appservice_id TEXT NOT NULL, + network_id TEXT NOT NULL, + room_id TEXT NOT NULL +); +CREATE UNIQUE INDEX appservice_room_list_idx ON appservice_room_list( + appservice_id, + network_id, + room_id +); +CREATE INDEX device_federation_outbox_id ON device_federation_outbox( + stream_id +); +CREATE TABLE federation_stream_position( + type TEXT NOT NULL, + stream_id INTEGER NOT NULL +); +CREATE TABLE device_lists_remote_cache( + user_id TEXT NOT NULL, + device_id TEXT NOT NULL, + content TEXT NOT NULL +); +CREATE TABLE device_lists_remote_extremeties( + user_id TEXT NOT NULL, + stream_id TEXT NOT NULL +); +CREATE TABLE device_lists_stream( + stream_id BIGINT NOT NULL, + user_id TEXT NOT NULL, + device_id TEXT NOT NULL +); +CREATE INDEX device_lists_stream_id ON device_lists_stream(stream_id, user_id); +CREATE TABLE device_lists_outbound_pokes( + destination TEXT NOT NULL, + stream_id BIGINT NOT NULL, + user_id TEXT NOT NULL, + device_id TEXT NOT NULL, + sent BOOLEAN NOT NULL, + ts BIGINT NOT NULL +); +CREATE INDEX device_lists_outbound_pokes_id ON device_lists_outbound_pokes( + destination, + stream_id +); +CREATE INDEX device_lists_outbound_pokes_user ON device_lists_outbound_pokes( + destination, + user_id +); +CREATE TABLE event_push_summary( + user_id TEXT NOT NULL, + room_id TEXT NOT NULL, + notif_count BIGINT NOT NULL, + stream_ordering BIGINT NOT NULL +); +CREATE INDEX event_push_summary_user_rm ON event_push_summary( + user_id, + room_id +); +CREATE TABLE event_push_summary_stream_ordering( + Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, + stream_ordering BIGINT NOT NULL, + CHECK(Lock='X') +); +CREATE TABLE IF NOT EXISTS "pushers"( + id BIGINT PRIMARY KEY, + user_name TEXT NOT NULL, + access_token BIGINT DEFAULT NULL, + profile_tag TEXT NOT NULL, + kind TEXT NOT NULL, + app_id TEXT NOT NULL, + app_display_name TEXT NOT NULL, + device_display_name TEXT NOT NULL, + pushkey TEXT NOT NULL, + ts BIGINT NOT NULL, + lang TEXT, + data TEXT, + last_stream_ordering INTEGER, + last_success BIGINT, + failing_since BIGINT, + UNIQUE(app_id, pushkey, user_name) +); +CREATE INDEX device_lists_outbound_pokes_stream ON device_lists_outbound_pokes( + stream_id +); +CREATE TABLE ratelimit_override( + user_id TEXT NOT NULL, + messages_per_second BIGINT, + burst_count BIGINT +); +CREATE UNIQUE INDEX ratelimit_override_idx ON ratelimit_override(user_id); +CREATE TABLE current_state_delta_stream( + stream_id BIGINT NOT NULL, + room_id TEXT NOT NULL, + type TEXT NOT NULL, + state_key TEXT NOT NULL, + event_id TEXT, + prev_event_id TEXT +); +CREATE INDEX current_state_delta_stream_idx ON current_state_delta_stream( + stream_id +); +CREATE TABLE device_lists_outbound_last_success( + destination TEXT NOT NULL, + user_id TEXT NOT NULL, + stream_id BIGINT NOT NULL +); +CREATE INDEX device_lists_outbound_last_success_idx ON device_lists_outbound_last_success( + destination, + user_id, + stream_id +); +CREATE TABLE user_directory_stream_pos( + Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, + stream_id BIGINT, + CHECK(Lock='X') +); +CREATE TABLE IF NOT EXISTS "users_in_public_rooms"( + user_id TEXT NOT NULL, + room_id TEXT NOT NULL +); +CREATE VIRTUAL TABLE user_directory_search USING fts4( + user_id, + value +) +/* user_directory_search( + user_id, + value +) */; +CREATE TABLE IF NOT EXISTS 'user_directory_search_content'( + docid INTEGER PRIMARY KEY, + 'c0user_id', + 'c1value' +); +CREATE TABLE IF NOT EXISTS 'user_directory_search_segments'( + blockid INTEGER PRIMARY KEY, + block BLOB +); +CREATE TABLE IF NOT EXISTS 'user_directory_search_segdir'( + level INTEGER, + idx INTEGER, + start_block INTEGER, + leaves_end_block INTEGER, + end_block INTEGER, + root BLOB, + PRIMARY KEY(level, idx) +); +CREATE TABLE IF NOT EXISTS 'user_directory_search_docsize'( + docid INTEGER PRIMARY KEY, + size BLOB +); +CREATE TABLE IF NOT EXISTS 'user_directory_search_stat'(id INTEGER PRIMARY KEY, value BLOB); +CREATE TABLE blocked_rooms(room_id TEXT NOT NULL, user_id TEXT NOT NULL); +CREATE UNIQUE INDEX blocked_rooms_idx ON blocked_rooms(room_id); +CREATE TABLE users_who_share_rooms( + user_id TEXT NOT NULL, + other_user_id TEXT NOT NULL, + room_id TEXT NOT NULL, + share_private BOOLEAN NOT NULL +); +CREATE UNIQUE INDEX users_who_share_rooms_u_idx ON users_who_share_rooms( + user_id, + other_user_id +); +CREATE INDEX users_who_share_rooms_r_idx ON users_who_share_rooms(room_id); +CREATE INDEX users_who_share_rooms_o_idx ON users_who_share_rooms( + other_user_id +); +CREATE TABLE IF NOT EXISTS "local_media_repository_url_cache"( + url TEXT, + response_code INTEGER, + etag TEXT, + expires_ts BIGINT, + og TEXT, + media_id TEXT, + download_ts BIGINT +); +CREATE INDEX local_media_repository_url_cache_expires_idx ON local_media_repository_url_cache( + expires_ts +); +CREATE INDEX local_media_repository_url_cache_by_url_download_ts ON local_media_repository_url_cache( + url, + download_ts +); +CREATE INDEX local_media_repository_url_cache_media_idx ON local_media_repository_url_cache( + media_id +); +CREATE TABLE group_users( + group_id TEXT NOT NULL, + user_id TEXT NOT NULL, + is_admin BOOLEAN NOT NULL, + is_public BOOLEAN NOT NULL +); +CREATE TABLE group_invites(group_id TEXT NOT NULL, user_id TEXT NOT NULL); +CREATE TABLE group_rooms( + group_id TEXT NOT NULL, + room_id TEXT NOT NULL, + is_public BOOLEAN NOT NULL +); +CREATE TABLE group_summary_rooms( + group_id TEXT NOT NULL, + room_id TEXT NOT NULL, + category_id TEXT NOT NULL, + room_order BIGINT NOT NULL, + is_public BOOLEAN NOT NULL, + UNIQUE(group_id, category_id, room_id, room_order), + CHECK(room_order > 0) +); +CREATE UNIQUE INDEX group_summary_rooms_g_idx ON group_summary_rooms( + group_id, + room_id, + category_id +); +CREATE TABLE group_summary_room_categories( + group_id TEXT NOT NULL, + category_id TEXT NOT NULL, + cat_order BIGINT NOT NULL, + UNIQUE(group_id, category_id, cat_order), + CHECK(cat_order > 0) +); +CREATE TABLE group_room_categories( + group_id TEXT NOT NULL, + category_id TEXT NOT NULL, + profile TEXT NOT NULL, + is_public BOOLEAN NOT NULL, + UNIQUE(group_id, category_id) +); +CREATE TABLE group_summary_users( + group_id TEXT NOT NULL, + user_id TEXT NOT NULL, + role_id TEXT NOT NULL, + user_order BIGINT NOT NULL, + is_public BOOLEAN NOT NULL +); +CREATE INDEX group_summary_users_g_idx ON group_summary_users(group_id); +CREATE TABLE group_summary_roles( + group_id TEXT NOT NULL, + role_id TEXT NOT NULL, + role_order BIGINT NOT NULL, + UNIQUE(group_id, role_id, role_order), + CHECK(role_order > 0) +); +CREATE TABLE group_roles( + group_id TEXT NOT NULL, + role_id TEXT NOT NULL, + profile TEXT NOT NULL, + is_public BOOLEAN NOT NULL, + UNIQUE(group_id, role_id) +); +CREATE TABLE group_attestations_renewals( + group_id TEXT NOT NULL, + user_id TEXT NOT NULL, + valid_until_ms BIGINT NOT NULL +); +CREATE INDEX group_attestations_renewals_g_idx ON group_attestations_renewals( + group_id, + user_id +); +CREATE INDEX group_attestations_renewals_u_idx ON group_attestations_renewals( + user_id +); +CREATE INDEX group_attestations_renewals_v_idx ON group_attestations_renewals( + valid_until_ms +); +CREATE TABLE group_attestations_remote( + group_id TEXT NOT NULL, + user_id TEXT NOT NULL, + valid_until_ms BIGINT NOT NULL, + attestation_json TEXT NOT NULL +); +CREATE INDEX group_attestations_remote_g_idx ON group_attestations_remote( + group_id, + user_id +); +CREATE INDEX group_attestations_remote_u_idx ON group_attestations_remote( + user_id +); +CREATE INDEX group_attestations_remote_v_idx ON group_attestations_remote( + valid_until_ms +); +CREATE TABLE local_group_membership( + group_id TEXT NOT NULL, + user_id TEXT NOT NULL, + is_admin BOOLEAN NOT NULL, + membership TEXT NOT NULL, + is_publicised BOOLEAN NOT NULL, + content TEXT NOT NULL +); +CREATE INDEX local_group_membership_u_idx ON local_group_membership( + user_id, + group_id +); +CREATE INDEX local_group_membership_g_idx ON local_group_membership(group_id); +CREATE TABLE local_group_updates( + stream_id BIGINT NOT NULL, + group_id TEXT NOT NULL, + user_id TEXT NOT NULL, + type TEXT NOT NULL, + content TEXT NOT NULL +); +CREATE TABLE remote_profile_cache( + user_id TEXT NOT NULL, + displayname TEXT, + avatar_url TEXT, + last_check BIGINT NOT NULL +); +CREATE UNIQUE INDEX remote_profile_cache_user_id ON remote_profile_cache( + user_id +); +CREATE INDEX remote_profile_cache_time ON remote_profile_cache(last_check); +CREATE TABLE IF NOT EXISTS "deleted_pushers"( + stream_id BIGINT NOT NULL, + app_id TEXT NOT NULL, + pushkey TEXT NOT NULL, + user_id TEXT NOT NULL +); +CREATE INDEX deleted_pushers_stream_id ON deleted_pushers(stream_id); +CREATE TABLE IF NOT EXISTS "groups"( + group_id TEXT NOT NULL, + name TEXT, + avatar_url TEXT, + short_description TEXT, + long_description TEXT, + is_public BOOL NOT NULL , + join_policy TEXT NOT NULL DEFAULT 'invite' +); +CREATE UNIQUE INDEX groups_idx ON groups(group_id); +CREATE TABLE IF NOT EXISTS "user_directory"( + user_id TEXT NOT NULL, + room_id TEXT, + display_name TEXT, + avatar_url TEXT +); +CREATE INDEX user_directory_room_idx ON user_directory(room_id); +CREATE UNIQUE INDEX user_directory_user_idx ON user_directory(user_id); +CREATE INDEX users_in_public_rooms_room_idx ON users_in_public_rooms(room_id); +CREATE UNIQUE INDEX users_in_public_rooms_user_idx ON users_in_public_rooms( + user_id +); +CREATE TABLE event_push_actions_staging( + event_id TEXT NOT NULL, + user_id TEXT NOT NULL, + actions TEXT NOT NULL, + notif SMALLINT NOT NULL, + highlight SMALLINT NOT NULL +); +CREATE INDEX event_push_actions_staging_id ON event_push_actions_staging( + event_id +); +CREATE TABLE users_pending_deactivation(user_id TEXT NOT NULL); +CREATE UNIQUE INDEX group_invites_g_idx ON group_invites(group_id, user_id); +CREATE UNIQUE INDEX group_users_g_idx ON group_users(group_id, user_id); +CREATE INDEX group_users_u_idx ON group_users(user_id); +CREATE INDEX group_invites_u_idx ON group_invites(user_id); +CREATE UNIQUE INDEX group_rooms_g_idx ON group_rooms(group_id, room_id); +CREATE INDEX group_rooms_r_idx ON group_rooms(room_id); +CREATE TABLE user_daily_visits( + user_id TEXT NOT NULL, + device_id TEXT, + timestamp BIGINT NOT NULL +); +CREATE INDEX user_daily_visits_uts_idx ON user_daily_visits( + user_id, + timestamp +); +CREATE INDEX user_daily_visits_ts_idx ON user_daily_visits(timestamp); +CREATE TABLE erased_users(user_id TEXT NOT NULL); +CREATE UNIQUE INDEX erased_users_user ON erased_users(user_id); +CREATE TABLE monthly_active_users( + user_id TEXT NOT NULL, + timestamp BIGINT NOT NULL +); +CREATE UNIQUE INDEX monthly_active_users_users ON monthly_active_users( + user_id +); +CREATE INDEX monthly_active_users_time_stamp ON monthly_active_users( + timestamp +); +CREATE TABLE IF NOT EXISTS "e2e_room_keys_versions"( + user_id TEXT NOT NULL, + version BIGINT NOT NULL, + algorithm TEXT NOT NULL, + auth_data TEXT NOT NULL, + deleted SMALLINT DEFAULT 0 NOT NULL +); +CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions( + user_id, + version +); +CREATE TABLE IF NOT EXISTS "e2e_room_keys"( + user_id TEXT NOT NULL, + room_id TEXT NOT NULL, + session_id TEXT NOT NULL, + version BIGINT NOT NULL, + first_message_index INT, + forwarded_count INT, + is_verified BOOLEAN, + session_data TEXT NOT NULL +); +CREATE UNIQUE INDEX e2e_room_keys_idx ON e2e_room_keys( + user_id, + room_id, + session_id +); +CREATE INDEX access_tokens_device_id ON access_tokens(user_id, device_id); +CREATE INDEX user_ips_device_id ON user_ips(user_id, device_id, last_seen); +CREATE INDEX event_contains_url_index ON events( + room_id, + topological_ordering, + stream_ordering +); +CREATE INDEX event_push_actions_u_highlight ON event_push_actions( + user_id, + stream_ordering +); +CREATE INDEX event_push_actions_highlights_index ON event_push_actions( + user_id, + room_id, + topological_ordering, + stream_ordering +); +CREATE INDEX current_state_events_member_index ON current_state_events( + state_key +); +CREATE INDEX device_inbox_stream_id_user_id ON device_inbox( + stream_id, + user_id +); +CREATE INDEX device_lists_stream_user_id ON device_lists_stream( + user_id, + device_id +); +CREATE INDEX local_media_repository_url_idx ON local_media_repository( + created_ts +); +CREATE INDEX user_ips_last_seen ON user_ips(user_id, last_seen); +CREATE INDEX user_ips_last_seen_only ON user_ips(last_seen); +CREATE INDEX users_creation_ts ON users(creation_ts); +CREATE INDEX event_to_state_groups_sg_index ON event_to_state_groups( + state_group +); +CREATE UNIQUE INDEX device_lists_remote_cache_unique_id ON device_lists_remote_cache( + user_id, + device_id +); +CREATE UNIQUE INDEX user_ips_device_unique_id ON user_ips( + user_id, + access_token, + ip, + user_agent, + device_id +); +CREATE INDEX state_groups_state_type_idx ON state_groups_state( + state_group, + type, + state_key +); +CREATE UNIQUE INDEX device_lists_remote_extremeties_unique_idx ON device_lists_remote_extremeties( + user_id +); diff --git a/synapse/storage/schema/full_schemas/README.txt b/synapse/storage/schema/full_schemas/README.txt new file mode 100644 index 000000000000..a4ad0c149492 --- /dev/null +++ b/synapse/storage/schema/full_schemas/README.txt @@ -0,0 +1,16 @@ +Building full schema dumps +========================== + +Postgres +-------- + +$ pg_dump --schema-only --no-comments --no-tablespaces --no-acl --no-owner ${DATABASE_NAME} > full.sql.postgres + +In the resulting file, delete all SET commands from the top of the file. + +SQLite +------ + +$ sqlite3 $DATABASE_FILE ".schema" > full.sql.sqlite + +Delete the CREATE statements for "schema_version", "applied_schema_deltas", and "applied_module_schemas". \ No newline at end of file diff --git a/tests/unittest.py b/tests/unittest.py index 6589196d855a..7273c73ce038 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -96,7 +96,7 @@ def __init__(self, methodName, *args, **kwargs): method = getattr(self, methodName) - level = getattr(method, "loglevel", getattr(self, "loglevel", logging.ERROR)) + level = getattr(method, "loglevel", getattr(self, "loglevel", logging.DEBUG)) @around(self) def setUp(orig): From a08154ab27c8ef76f89d0634d91320180596c342 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 4 Jan 2019 03:42:41 +1100 Subject: [PATCH 29/40] add the inserts :/ --- synapse/storage/prepare_database.py | 2 +- .../storage/schema/full_schemas/54/zzz-inserts.sql | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/schema/full_schemas/54/zzz-inserts.sql diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 3d79c3d50fc6..535c0666f784 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -146,7 +146,7 @@ def _setup_new_database(cur, database_engine): directory_entries = os.listdir(sql_dir) - for filename in fnmatch.filter(directory_entries, "*.sql") + fnmatch.filter(directory_entries, "*.sql." + specific): + for filename in sorted(fnmatch.filter(directory_entries, "*.sql") + fnmatch.filter(directory_entries, "*.sql." + specific)): sql_loc = os.path.join(sql_dir, filename) logger.debug("Applying schema %s", sql_loc) executescript(cur, sql_loc) diff --git a/synapse/storage/schema/full_schemas/54/zzz-inserts.sql b/synapse/storage/schema/full_schemas/54/zzz-inserts.sql new file mode 100644 index 000000000000..859691b0fb75 --- /dev/null +++ b/synapse/storage/schema/full_schemas/54/zzz-inserts.sql @@ -0,0 +1,12 @@ +INSERT INTO user_directory_stream_pos (stream_id) VALUES (null); + +INSERT INTO appservice_stream_position (stream_ordering) + SELECT COALESCE(MAX(stream_ordering), 0) FROM events; + +INSERT INTO device_max_stream_id (stream_id) + SELECT COALESCE(MAX(stream_id), 0) FROM device_inbox; + +INSERT INTO event_push_summary_stream_ordering (stream_ordering) VALUES (0); + +INSERT INTO federation_stream_position (type, stream_id) VALUES ('federation', -1); +INSERT INTO federation_stream_position (type, stream_id) SELECT 'events', coalesce(max(stream_ordering), -1) FROM events; From 9ead35180c42add2a43f4f1640dd69dc3aae107d Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 4 Jan 2019 04:03:16 +1100 Subject: [PATCH 30/40] clean up packaging --- MANIFEST.in | 2 ++ synapse/storage/prepare_database.py | 2 ++ synapse/storage/schema/full_schemas/{README.txt => README.md} | 0 3 files changed, 4 insertions(+) rename synapse/storage/schema/full_schemas/{README.txt => README.md} (100%) diff --git a/MANIFEST.in b/MANIFEST.in index 29303cc8b57c..2c023fd7a453 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,7 +9,9 @@ include demo/*.py include demo/*.sh recursive-include synapse/storage/schema *.sql +recursive-include synapse/storage/schema *.sql.* recursive-include synapse/storage/schema *.py +recursive-include synapse/storage/schema *.md recursive-include docs * recursive-include scripts * diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 535c0666f784..5f123436a468 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -132,6 +132,8 @@ def _setup_new_database(cur, database_engine): ver = int(match.group(0)) if ver <= SCHEMA_VERSION: valid_dirs.append((ver, abs_path)) + elif filename == "README.md": + # Ignore the readme else: logger.warn("Unexpected entry in 'full_schemas': %s", filename) diff --git a/synapse/storage/schema/full_schemas/README.txt b/synapse/storage/schema/full_schemas/README.md similarity index 100% rename from synapse/storage/schema/full_schemas/README.txt rename to synapse/storage/schema/full_schemas/README.md From 1c9ef51938ec9cdcbe9774bf1c4fb26cf38d5b4f Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 4 Jan 2019 04:06:22 +1100 Subject: [PATCH 31/40] cleanups --- synapse/storage/_base.py | 5 ++++- synapse/storage/client_ips.py | 4 +++- synapse/storage/prepare_database.py | 6 +++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 50606f8675a2..f8ab3b07bd8d 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -603,7 +603,10 @@ def _simple_upsert_txn( Pick the UPSERT method which works best on the platform. Either the native one (Pg9.5+, recent SQLites), or fall back to an emulated method. """ - if self.database_engine.can_native_upsert and table not in self._unsafe_to_upsert_tables: + if ( + self.database_engine.can_native_upsert + and table not in self._unsafe_to_upsert_tables + ): return self._simple_upsert_txn_native_upsert( txn, table, diff --git a/synapse/storage/client_ips.py b/synapse/storage/client_ips.py index e4261d78cd6e..8760882770a3 100644 --- a/synapse/storage/client_ips.py +++ b/synapse/storage/client_ips.py @@ -123,7 +123,9 @@ def update(): ) def _update_client_ips_batch_txn(self, txn, to_update): - if "user_ips" in self._unsafe_to_upsert_tables or (not self.database_engine.can_native_upsert): + if "user_ips" in self._unsafe_to_upsert_tables or ( + not self.database_engine.can_native_upsert + ): self.database_engine.lock_table(txn, "user_ips") for entry in iteritems(to_update): diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 5f123436a468..48dd64e9a0ec 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -134,6 +134,7 @@ def _setup_new_database(cur, database_engine): valid_dirs.append((ver, abs_path)) elif filename == "README.md": # Ignore the readme + pass else: logger.warn("Unexpected entry in 'full_schemas': %s", filename) @@ -148,7 +149,10 @@ def _setup_new_database(cur, database_engine): directory_entries = os.listdir(sql_dir) - for filename in sorted(fnmatch.filter(directory_entries, "*.sql") + fnmatch.filter(directory_entries, "*.sql." + specific)): + for filename in sorted( + fnmatch.filter(directory_entries, "*.sql") + + fnmatch.filter(directory_entries, "*.sql." + specific) + ): sql_loc = os.path.join(sql_dir, filename) logger.debug("Applying schema %s", sql_loc) executescript(cur, sql_loc) From 4e66437e841c82fc98752d338f726273dc968fd1 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 4 Jan 2019 04:07:00 +1100 Subject: [PATCH 32/40] cleanups --- synapse/storage/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index f8ab3b07bd8d..1b87e9cf02f0 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -220,7 +220,7 @@ def _check_safe_to_upsert(self): # The User IPs table in schema #53 was missing a unique index, which we # run as a background update. - if not "user_ips_device_unique_index" in updates: + if "user_ips_device_unique_index" not in updates: self._unsafe_to_upsert_tables.discard("user_id") # If there's any tables left to check, reschedule to run. From b6d7fed2c88ac67d2af47ea3639ba66d887188e4 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 18 Jan 2019 17:17:41 +1100 Subject: [PATCH 33/40] things we don't need --- synapse/storage/event_push_actions.py | 20 +- .../schema/delta/54/user_ips_index.sql | 18 - .../schema/full_schemas/54/full.sql.postgres | 3186 ----------------- .../schema/full_schemas/54/full.sql.sqlite | 1203 ------- .../schema/full_schemas/54/zzz-inserts.sql | 12 - 5 files changed, 6 insertions(+), 4433 deletions(-) delete mode 100644 synapse/storage/schema/delta/54/user_ips_index.sql delete mode 100644 synapse/storage/schema/full_schemas/54/full.sql.postgres delete mode 100644 synapse/storage/schema/full_schemas/54/full.sql.sqlite delete mode 100644 synapse/storage/schema/full_schemas/54/zzz-inserts.sql diff --git a/synapse/storage/event_push_actions.py b/synapse/storage/event_push_actions.py index 371e8f7f3eed..6840320641da 100644 --- a/synapse/storage/event_push_actions.py +++ b/synapse/storage/event_push_actions.py @@ -22,7 +22,6 @@ from twisted.internet import defer -from synapse.api.errors import StoreError from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage._base import LoggingTransaction, SQLBaseStore from synapse.util.caches.descriptors import cachedInlineCallbacks @@ -851,19 +850,12 @@ def _rotate_notifs_txn(self, txn): return caught_up def _rotate_notifs_before_txn(self, txn, rotate_to_stream_ordering): - try: - old_rotate_stream_ordering = self._simple_select_one_onecol_txn( - txn, - table="event_push_summary_stream_ordering", - keyvalues={}, - retcol="stream_ordering", - ) - except StoreError as e: - if e.code == 404: - # The database is empty, assume 0. - old_rotate_stream_ordering = 0 - else: - raise + old_rotate_stream_ordering = self._simple_select_one_onecol_txn( + txn, + table="event_push_summary_stream_ordering", + keyvalues={}, + retcol="stream_ordering", + ) # Calculate the new counts that should be upserted into event_push_summary sql = """ diff --git a/synapse/storage/schema/delta/54/user_ips_index.sql b/synapse/storage/schema/delta/54/user_ips_index.sql deleted file mode 100644 index 292bcbe8e998..000000000000 --- a/synapse/storage/schema/delta/54/user_ips_index.sql +++ /dev/null @@ -1,18 +0,0 @@ -/* Copyright 2018 New Vector Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - --- add a new unique index to user_ips table -INSERT INTO background_updates (update_name, progress_json) VALUES - ('user_ips_device_unique_index', '{}'); diff --git a/synapse/storage/schema/full_schemas/54/full.sql.postgres b/synapse/storage/schema/full_schemas/54/full.sql.postgres deleted file mode 100644 index 245db5aff0b1..000000000000 --- a/synapse/storage/schema/full_schemas/54/full.sql.postgres +++ /dev/null @@ -1,3186 +0,0 @@ --- --- PostgreSQL database dump --- - --- Dumped from database version 11.1 --- Dumped by pg_dump version 11.1 - --- --- Name: access_tokens; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE access_tokens ( - id bigint NOT NULL, - user_id text NOT NULL, - device_id text, - token text NOT NULL, - last_used bigint -); - - --- --- Name: account_data; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE account_data ( - user_id text NOT NULL, - account_data_type text NOT NULL, - stream_id bigint NOT NULL, - content text NOT NULL -); - - --- --- Name: account_data_max_stream_id; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE account_data_max_stream_id ( - lock character(1) DEFAULT 'X'::bpchar NOT NULL, - stream_id bigint NOT NULL, - CONSTRAINT private_user_data_max_stream_id_lock_check CHECK ((lock = 'X'::bpchar)) -); - - --- --- Name: application_services; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE application_services ( - id bigint NOT NULL, - url text, - token text, - hs_token text, - sender text -); - - --- --- Name: application_services_regex; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE application_services_regex ( - id bigint NOT NULL, - as_id bigint NOT NULL, - namespace integer, - regex text -); - - --- --- Name: application_services_state; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE application_services_state ( - as_id text NOT NULL, - state character varying(5), - last_txn integer -); - - --- --- Name: application_services_txns; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE application_services_txns ( - as_id text NOT NULL, - txn_id integer NOT NULL, - event_ids text NOT NULL -); - - --- --- Name: appservice_room_list; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE appservice_room_list ( - appservice_id text NOT NULL, - network_id text NOT NULL, - room_id text NOT NULL -); - - --- --- Name: appservice_stream_position; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE appservice_stream_position ( - lock character(1) DEFAULT 'X'::bpchar NOT NULL, - stream_ordering bigint, - CONSTRAINT appservice_stream_position_lock_check CHECK ((lock = 'X'::bpchar)) -); - - --- --- Name: background_updates; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE background_updates ( - update_name text NOT NULL, - progress_json text NOT NULL, - depends_on text -); - - --- --- Name: blocked_rooms; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE blocked_rooms ( - room_id text NOT NULL, - user_id text NOT NULL -); - - --- --- Name: cache_invalidation_stream; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE cache_invalidation_stream ( - stream_id bigint, - cache_func text, - keys text[], - invalidation_ts bigint -); - - --- --- Name: current_state_delta_stream; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE current_state_delta_stream ( - stream_id bigint NOT NULL, - room_id text NOT NULL, - type text NOT NULL, - state_key text NOT NULL, - event_id text, - prev_event_id text -); - - --- --- Name: current_state_events; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE current_state_events ( - event_id text NOT NULL, - room_id text NOT NULL, - type text NOT NULL, - state_key text NOT NULL -); - - --- --- Name: current_state_resets; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE current_state_resets ( - event_stream_ordering bigint NOT NULL -); - - --- --- Name: deleted_pushers; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE deleted_pushers ( - stream_id bigint NOT NULL, - app_id text NOT NULL, - pushkey text NOT NULL, - user_id text NOT NULL -); - - --- --- Name: destinations; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE destinations ( - destination text NOT NULL, - retry_last_ts bigint, - retry_interval integer -); - - --- --- Name: device_federation_inbox; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE device_federation_inbox ( - origin text NOT NULL, - message_id text NOT NULL, - received_ts bigint NOT NULL -); - - --- --- Name: device_federation_outbox; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE device_federation_outbox ( - destination text NOT NULL, - stream_id bigint NOT NULL, - queued_ts bigint NOT NULL, - messages_json text NOT NULL -); - - --- --- Name: device_inbox; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE device_inbox ( - user_id text NOT NULL, - device_id text NOT NULL, - stream_id bigint NOT NULL, - message_json text NOT NULL -); - - --- --- Name: device_lists_outbound_last_success; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE device_lists_outbound_last_success ( - destination text NOT NULL, - user_id text NOT NULL, - stream_id bigint NOT NULL -); - - --- --- Name: device_lists_outbound_pokes; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE device_lists_outbound_pokes ( - destination text NOT NULL, - stream_id bigint NOT NULL, - user_id text NOT NULL, - device_id text NOT NULL, - sent boolean NOT NULL, - ts bigint NOT NULL -); - - --- --- Name: device_lists_remote_cache; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE device_lists_remote_cache ( - user_id text NOT NULL, - device_id text NOT NULL, - content text NOT NULL -); - - --- --- Name: device_lists_remote_extremeties; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE device_lists_remote_extremeties ( - user_id text NOT NULL, - stream_id text NOT NULL -); - - --- --- Name: device_lists_stream; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE device_lists_stream ( - stream_id bigint NOT NULL, - user_id text NOT NULL, - device_id text NOT NULL -); - - --- --- Name: device_max_stream_id; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE device_max_stream_id ( - stream_id bigint NOT NULL -); - - --- --- Name: devices; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE devices ( - user_id text NOT NULL, - device_id text NOT NULL, - display_name text -); - - --- --- Name: e2e_device_keys_json; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE e2e_device_keys_json ( - user_id text NOT NULL, - device_id text NOT NULL, - ts_added_ms bigint NOT NULL, - key_json text NOT NULL -); - - --- --- Name: e2e_one_time_keys_json; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE e2e_one_time_keys_json ( - user_id text NOT NULL, - device_id text NOT NULL, - algorithm text NOT NULL, - key_id text NOT NULL, - ts_added_ms bigint NOT NULL, - key_json text NOT NULL -); - - --- --- Name: e2e_room_keys; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE e2e_room_keys ( - user_id text NOT NULL, - room_id text NOT NULL, - session_id text NOT NULL, - version bigint NOT NULL, - first_message_index integer, - forwarded_count integer, - is_verified boolean, - session_data text NOT NULL -); - - --- --- Name: e2e_room_keys_versions; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE e2e_room_keys_versions ( - user_id text NOT NULL, - version bigint NOT NULL, - algorithm text NOT NULL, - auth_data text NOT NULL, - deleted smallint DEFAULT 0 NOT NULL -); - - --- --- Name: erased_users; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE erased_users ( - user_id text NOT NULL -); - - --- --- Name: event_auth; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE event_auth ( - event_id text NOT NULL, - auth_id text NOT NULL, - room_id text NOT NULL -); - - --- --- Name: event_backward_extremities; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE event_backward_extremities ( - event_id text NOT NULL, - room_id text NOT NULL -); - - --- --- Name: event_content_hashes; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE event_content_hashes ( - event_id text, - algorithm text, - hash bytea -); - - --- --- Name: event_destinations; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE event_destinations ( - event_id text NOT NULL, - destination text NOT NULL, - delivered_ts bigint DEFAULT 0 -); - - --- --- Name: event_edge_hashes; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE event_edge_hashes ( - event_id text, - prev_event_id text, - algorithm text, - hash bytea -); - - --- --- Name: event_edges; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE event_edges ( - event_id text NOT NULL, - prev_event_id text NOT NULL, - room_id text NOT NULL, - is_state boolean NOT NULL -); - - --- --- Name: event_forward_extremities; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE event_forward_extremities ( - event_id text NOT NULL, - room_id text NOT NULL -); - - --- --- Name: event_json; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE event_json ( - event_id text NOT NULL, - room_id text NOT NULL, - internal_metadata text NOT NULL, - json text NOT NULL -); - - --- --- Name: event_push_actions; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE event_push_actions ( - room_id text NOT NULL, - event_id text NOT NULL, - user_id text NOT NULL, - profile_tag character varying(32), - actions text NOT NULL, - topological_ordering bigint, - stream_ordering bigint, - notif smallint, - highlight smallint -); - - --- --- Name: event_push_actions_staging; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE event_push_actions_staging ( - event_id text NOT NULL, - user_id text NOT NULL, - actions text NOT NULL, - notif smallint NOT NULL, - highlight smallint NOT NULL -); - - --- --- Name: event_push_summary; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE event_push_summary ( - user_id text NOT NULL, - room_id text NOT NULL, - notif_count bigint NOT NULL, - stream_ordering bigint NOT NULL -); - - --- --- Name: event_push_summary_stream_ordering; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE event_push_summary_stream_ordering ( - lock character(1) DEFAULT 'X'::bpchar NOT NULL, - stream_ordering bigint NOT NULL, - CONSTRAINT event_push_summary_stream_ordering_lock_check CHECK ((lock = 'X'::bpchar)) -); - - --- --- Name: event_reference_hashes; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE event_reference_hashes ( - event_id text, - algorithm text, - hash bytea -); - - --- --- Name: event_reports; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE event_reports ( - id bigint NOT NULL, - received_ts bigint NOT NULL, - room_id text NOT NULL, - event_id text NOT NULL, - user_id text NOT NULL, - reason text, - content text -); - - --- --- Name: event_search; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE event_search ( - event_id text, - room_id text, - sender text, - key text, - vector tsvector, - origin_server_ts bigint, - stream_ordering bigint -); - - --- --- Name: event_signatures; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE event_signatures ( - event_id text, - signature_name text, - key_id text, - signature bytea -); - - --- --- Name: event_to_state_groups; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE event_to_state_groups ( - event_id text NOT NULL, - state_group bigint NOT NULL -); - - --- --- Name: events; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE events ( - stream_ordering integer NOT NULL, - topological_ordering bigint NOT NULL, - event_id text NOT NULL, - type text NOT NULL, - room_id text NOT NULL, - content text, - unrecognized_keys text, - processed boolean NOT NULL, - outlier boolean NOT NULL, - depth bigint DEFAULT 0 NOT NULL, - origin_server_ts bigint, - received_ts bigint, - sender text, - contains_url boolean -); - - --- --- Name: ex_outlier_stream; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE ex_outlier_stream ( - event_stream_ordering bigint NOT NULL, - event_id text NOT NULL, - state_group bigint NOT NULL -); - - --- --- Name: federation_stream_position; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE federation_stream_position ( - type text NOT NULL, - stream_id integer NOT NULL -); - - --- --- Name: feedback; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE feedback ( - event_id text NOT NULL, - feedback_type text, - target_event_id text, - sender text, - room_id text -); - - --- --- Name: group_attestations_remote; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE group_attestations_remote ( - group_id text NOT NULL, - user_id text NOT NULL, - valid_until_ms bigint NOT NULL, - attestation_json text NOT NULL -); - - --- --- Name: group_attestations_renewals; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE group_attestations_renewals ( - group_id text NOT NULL, - user_id text NOT NULL, - valid_until_ms bigint NOT NULL -); - - --- --- Name: group_invites; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE group_invites ( - group_id text NOT NULL, - user_id text NOT NULL -); - - --- --- Name: group_roles; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE group_roles ( - group_id text NOT NULL, - role_id text NOT NULL, - profile text NOT NULL, - is_public boolean NOT NULL -); - - --- --- Name: group_room_categories; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE group_room_categories ( - group_id text NOT NULL, - category_id text NOT NULL, - profile text NOT NULL, - is_public boolean NOT NULL -); - - --- --- Name: group_rooms; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE group_rooms ( - group_id text NOT NULL, - room_id text NOT NULL, - is_public boolean NOT NULL -); - - --- --- Name: group_summary_roles; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE group_summary_roles ( - group_id text NOT NULL, - role_id text NOT NULL, - role_order bigint NOT NULL, - CONSTRAINT group_summary_roles_role_order_check CHECK ((role_order > 0)) -); - - --- --- Name: group_summary_room_categories; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE group_summary_room_categories ( - group_id text NOT NULL, - category_id text NOT NULL, - cat_order bigint NOT NULL, - CONSTRAINT group_summary_room_categories_cat_order_check CHECK ((cat_order > 0)) -); - - --- --- Name: group_summary_rooms; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE group_summary_rooms ( - group_id text NOT NULL, - room_id text NOT NULL, - category_id text NOT NULL, - room_order bigint NOT NULL, - is_public boolean NOT NULL, - CONSTRAINT group_summary_rooms_room_order_check CHECK ((room_order > 0)) -); - - --- --- Name: group_summary_users; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE group_summary_users ( - group_id text NOT NULL, - user_id text NOT NULL, - role_id text NOT NULL, - user_order bigint NOT NULL, - is_public boolean NOT NULL -); - - --- --- Name: group_users; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE group_users ( - group_id text NOT NULL, - user_id text NOT NULL, - is_admin boolean NOT NULL, - is_public boolean NOT NULL -); - - --- --- Name: groups; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE groups ( - group_id text NOT NULL, - name text, - avatar_url text, - short_description text, - long_description text, - is_public boolean NOT NULL, - join_policy text DEFAULT 'invite'::text NOT NULL -); - - --- --- Name: guest_access; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE guest_access ( - event_id text NOT NULL, - room_id text NOT NULL, - guest_access text NOT NULL -); - - --- --- Name: history_visibility; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE history_visibility ( - event_id text NOT NULL, - room_id text NOT NULL, - history_visibility text NOT NULL -); - - --- --- Name: local_group_membership; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE local_group_membership ( - group_id text NOT NULL, - user_id text NOT NULL, - is_admin boolean NOT NULL, - membership text NOT NULL, - is_publicised boolean NOT NULL, - content text NOT NULL -); - - --- --- Name: local_group_updates; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE local_group_updates ( - stream_id bigint NOT NULL, - group_id text NOT NULL, - user_id text NOT NULL, - type text NOT NULL, - content text NOT NULL -); - - --- --- Name: local_invites; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE local_invites ( - stream_id bigint NOT NULL, - inviter text NOT NULL, - invitee text NOT NULL, - event_id text NOT NULL, - room_id text NOT NULL, - locally_rejected text, - replaced_by text -); - - --- --- Name: local_media_repository; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE local_media_repository ( - media_id text, - media_type text, - media_length integer, - created_ts bigint, - upload_name text, - user_id text, - quarantined_by text, - url_cache text, - last_access_ts bigint -); - - --- --- Name: local_media_repository_thumbnails; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE local_media_repository_thumbnails ( - media_id text, - thumbnail_width integer, - thumbnail_height integer, - thumbnail_type text, - thumbnail_method text, - thumbnail_length integer -); - - --- --- Name: local_media_repository_url_cache; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE local_media_repository_url_cache ( - url text, - response_code integer, - etag text, - expires_ts bigint, - og text, - media_id text, - download_ts bigint -); - - --- --- Name: monthly_active_users; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE monthly_active_users ( - user_id text NOT NULL, - "timestamp" bigint NOT NULL -); - - --- --- Name: open_id_tokens; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE open_id_tokens ( - token text NOT NULL, - ts_valid_until_ms bigint NOT NULL, - user_id text NOT NULL -); - - --- --- Name: presence; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE presence ( - user_id text NOT NULL, - state character varying(20), - status_msg text, - mtime bigint -); - - --- --- Name: presence_allow_inbound; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE presence_allow_inbound ( - observed_user_id text NOT NULL, - observer_user_id text NOT NULL -); - - --- --- Name: presence_list; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE presence_list ( - user_id text NOT NULL, - observed_user_id text NOT NULL, - accepted boolean NOT NULL -); - - --- --- Name: presence_stream; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE presence_stream ( - stream_id bigint, - user_id text, - state text, - last_active_ts bigint, - last_federation_update_ts bigint, - last_user_sync_ts bigint, - status_msg text, - currently_active boolean -); - - --- --- Name: profiles; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE profiles ( - user_id text NOT NULL, - displayname text, - avatar_url text -); - - --- --- Name: public_room_list_stream; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public_room_list_stream ( - stream_id bigint NOT NULL, - room_id text NOT NULL, - visibility boolean NOT NULL, - appservice_id text, - network_id text -); - - --- --- Name: push_rules; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE push_rules ( - id bigint NOT NULL, - user_name text NOT NULL, - rule_id text NOT NULL, - priority_class smallint NOT NULL, - priority integer DEFAULT 0 NOT NULL, - conditions text NOT NULL, - actions text NOT NULL -); - - --- --- Name: push_rules_enable; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE push_rules_enable ( - id bigint NOT NULL, - user_name text NOT NULL, - rule_id text NOT NULL, - enabled smallint -); - - --- --- Name: push_rules_stream; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE push_rules_stream ( - stream_id bigint NOT NULL, - event_stream_ordering bigint NOT NULL, - user_id text NOT NULL, - rule_id text NOT NULL, - op text NOT NULL, - priority_class smallint, - priority integer, - conditions text, - actions text -); - - --- --- Name: pusher_throttle; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE pusher_throttle ( - pusher bigint NOT NULL, - room_id text NOT NULL, - last_sent_ts bigint, - throttle_ms bigint -); - - --- --- Name: pushers; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE pushers ( - id bigint NOT NULL, - user_name text NOT NULL, - access_token bigint, - profile_tag text NOT NULL, - kind text NOT NULL, - app_id text NOT NULL, - app_display_name text NOT NULL, - device_display_name text NOT NULL, - pushkey text NOT NULL, - ts bigint NOT NULL, - lang text, - data text, - last_stream_ordering integer, - last_success bigint, - failing_since bigint -); - - --- --- Name: ratelimit_override; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE ratelimit_override ( - user_id text NOT NULL, - messages_per_second bigint, - burst_count bigint -); - - --- --- Name: receipts_graph; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE receipts_graph ( - room_id text NOT NULL, - receipt_type text NOT NULL, - user_id text NOT NULL, - event_ids text NOT NULL, - data text NOT NULL -); - - --- --- Name: receipts_linearized; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE receipts_linearized ( - stream_id bigint NOT NULL, - room_id text NOT NULL, - receipt_type text NOT NULL, - user_id text NOT NULL, - event_id text NOT NULL, - data text NOT NULL -); - - --- --- Name: received_transactions; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE received_transactions ( - transaction_id text, - origin text, - ts bigint, - response_code integer, - response_json bytea, - has_been_referenced smallint DEFAULT 0 -); - - --- --- Name: redactions; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE redactions ( - event_id text NOT NULL, - redacts text NOT NULL -); - - --- --- Name: rejections; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE rejections ( - event_id text NOT NULL, - reason text NOT NULL, - last_check text NOT NULL -); - - --- --- Name: remote_media_cache; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE remote_media_cache ( - media_origin text, - media_id text, - media_type text, - created_ts bigint, - upload_name text, - media_length integer, - filesystem_id text, - last_access_ts bigint, - quarantined_by text -); - - --- --- Name: remote_media_cache_thumbnails; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE remote_media_cache_thumbnails ( - media_origin text, - media_id text, - thumbnail_width integer, - thumbnail_height integer, - thumbnail_method text, - thumbnail_type text, - thumbnail_length integer, - filesystem_id text -); - - --- --- Name: remote_profile_cache; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE remote_profile_cache ( - user_id text NOT NULL, - displayname text, - avatar_url text, - last_check bigint NOT NULL -); - - --- --- Name: room_account_data; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE room_account_data ( - user_id text NOT NULL, - room_id text NOT NULL, - account_data_type text NOT NULL, - stream_id bigint NOT NULL, - content text NOT NULL -); - - --- --- Name: room_alias_servers; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE room_alias_servers ( - room_alias text NOT NULL, - server text NOT NULL -); - - --- --- Name: room_aliases; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE room_aliases ( - room_alias text NOT NULL, - room_id text NOT NULL, - creator text -); - - --- --- Name: room_depth; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE room_depth ( - room_id text NOT NULL, - min_depth integer NOT NULL -); - - --- --- Name: room_hosts; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE room_hosts ( - room_id text NOT NULL, - host text NOT NULL -); - - --- --- Name: room_memberships; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE room_memberships ( - event_id text NOT NULL, - user_id text NOT NULL, - sender text NOT NULL, - room_id text NOT NULL, - membership text NOT NULL, - forgotten integer DEFAULT 0, - display_name text, - avatar_url text -); - - --- --- Name: room_names; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE room_names ( - event_id text NOT NULL, - room_id text NOT NULL, - name text NOT NULL -); - - --- --- Name: room_tags; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE room_tags ( - user_id text NOT NULL, - room_id text NOT NULL, - tag text NOT NULL, - content text NOT NULL -); - - --- --- Name: room_tags_revisions; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE room_tags_revisions ( - user_id text NOT NULL, - room_id text NOT NULL, - stream_id bigint NOT NULL -); - - --- --- Name: rooms; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE rooms ( - room_id text NOT NULL, - is_public boolean, - creator text -); - - --- --- Name: server_keys_json; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE server_keys_json ( - server_name text NOT NULL, - key_id text NOT NULL, - from_server text NOT NULL, - ts_added_ms bigint NOT NULL, - ts_valid_until_ms bigint NOT NULL, - key_json bytea NOT NULL -); - - --- --- Name: server_signature_keys; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE server_signature_keys ( - server_name text, - key_id text, - from_server text, - ts_added_ms bigint, - verify_key bytea -); - - --- --- Name: server_tls_certificates; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE server_tls_certificates ( - server_name text, - fingerprint text, - from_server text, - ts_added_ms bigint, - tls_certificate bytea -); - - --- --- Name: state_events; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE state_events ( - event_id text NOT NULL, - room_id text NOT NULL, - type text NOT NULL, - state_key text NOT NULL, - prev_state text -); - - --- --- Name: state_forward_extremities; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE state_forward_extremities ( - event_id text NOT NULL, - room_id text NOT NULL, - type text NOT NULL, - state_key text NOT NULL -); - - --- --- Name: state_group_edges; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE state_group_edges ( - state_group bigint NOT NULL, - prev_state_group bigint NOT NULL -); - - --- --- Name: state_group_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE state_group_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: state_groups; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE state_groups ( - id bigint NOT NULL, - room_id text NOT NULL, - event_id text NOT NULL -); - - --- --- Name: state_groups_state; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE state_groups_state ( - state_group bigint NOT NULL, - room_id text NOT NULL, - type text NOT NULL, - state_key text NOT NULL, - event_id text NOT NULL -); - - --- --- Name: stats_reporting; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE stats_reporting ( - reported_stream_token integer, - reported_time bigint -); - - --- --- Name: stream_ordering_to_exterm; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE stream_ordering_to_exterm ( - stream_ordering bigint NOT NULL, - room_id text NOT NULL, - event_id text NOT NULL -); - - --- --- Name: threepid_guest_access_tokens; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE threepid_guest_access_tokens ( - medium text, - address text, - guest_access_token text, - first_inviter text -); - - --- --- Name: topics; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE topics ( - event_id text NOT NULL, - room_id text NOT NULL, - topic text NOT NULL -); - - --- --- Name: transaction_id_to_pdu; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE transaction_id_to_pdu ( - transaction_id integer, - destination text, - pdu_id text, - pdu_origin text -); - - --- --- Name: user_daily_visits; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE user_daily_visits ( - user_id text NOT NULL, - device_id text, - "timestamp" bigint NOT NULL -); - - --- --- Name: user_directory; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE user_directory ( - user_id text NOT NULL, - room_id text, - display_name text, - avatar_url text -); - - --- --- Name: user_directory_search; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE user_directory_search ( - user_id text NOT NULL, - vector tsvector -); - - --- --- Name: user_directory_stream_pos; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE user_directory_stream_pos ( - lock character(1) DEFAULT 'X'::bpchar NOT NULL, - stream_id bigint, - CONSTRAINT user_directory_stream_pos_lock_check CHECK ((lock = 'X'::bpchar)) -); - - --- --- Name: user_filters; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE user_filters ( - user_id text, - filter_id bigint, - filter_json bytea -); - - --- --- Name: user_ips; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE user_ips ( - user_id text NOT NULL, - access_token text NOT NULL, - device_id text, - ip text NOT NULL, - user_agent text NOT NULL, - last_seen bigint NOT NULL -); - - --- --- Name: user_threepids; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE user_threepids ( - user_id text NOT NULL, - medium text NOT NULL, - address text NOT NULL, - validated_at bigint NOT NULL, - added_at bigint NOT NULL -); - - --- --- Name: users; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE users ( - name text, - password_hash text, - creation_ts bigint, - admin smallint DEFAULT 0 NOT NULL, - upgrade_ts bigint, - is_guest smallint DEFAULT 0 NOT NULL, - appservice_id text, - consent_version text, - consent_server_notice_sent text, - user_type text -); - - --- --- Name: users_in_public_rooms; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE users_in_public_rooms ( - user_id text NOT NULL, - room_id text NOT NULL -); - - --- --- Name: users_pending_deactivation; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE users_pending_deactivation ( - user_id text NOT NULL -); - - --- --- Name: users_who_share_rooms; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE users_who_share_rooms ( - user_id text NOT NULL, - other_user_id text NOT NULL, - room_id text NOT NULL, - share_private boolean NOT NULL -); - - --- --- Name: access_tokens access_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY access_tokens - ADD CONSTRAINT access_tokens_pkey PRIMARY KEY (id); - - --- --- Name: access_tokens access_tokens_token_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY access_tokens - ADD CONSTRAINT access_tokens_token_key UNIQUE (token); - - --- --- Name: account_data account_data_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY account_data - ADD CONSTRAINT account_data_uniqueness UNIQUE (user_id, account_data_type); - - --- --- Name: application_services application_services_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY application_services - ADD CONSTRAINT application_services_pkey PRIMARY KEY (id); - - --- --- Name: application_services_regex application_services_regex_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY application_services_regex - ADD CONSTRAINT application_services_regex_pkey PRIMARY KEY (id); - - --- --- Name: application_services_state application_services_state_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY application_services_state - ADD CONSTRAINT application_services_state_pkey PRIMARY KEY (as_id); - - --- --- Name: application_services application_services_token_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY application_services - ADD CONSTRAINT application_services_token_key UNIQUE (token); - - --- --- Name: application_services_txns application_services_txns_as_id_txn_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY application_services_txns - ADD CONSTRAINT application_services_txns_as_id_txn_id_key UNIQUE (as_id, txn_id); - - --- --- Name: appservice_stream_position appservice_stream_position_lock_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY appservice_stream_position - ADD CONSTRAINT appservice_stream_position_lock_key UNIQUE (lock); - - --- --- Name: background_updates background_updates_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY background_updates - ADD CONSTRAINT background_updates_uniqueness UNIQUE (update_name); - - --- --- Name: current_state_events current_state_events_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY current_state_events - ADD CONSTRAINT current_state_events_event_id_key UNIQUE (event_id); - - --- --- Name: current_state_events current_state_events_room_id_type_state_key_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY current_state_events - ADD CONSTRAINT current_state_events_room_id_type_state_key_key UNIQUE (room_id, type, state_key); - - --- --- Name: current_state_resets current_state_resets_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY current_state_resets - ADD CONSTRAINT current_state_resets_pkey PRIMARY KEY (event_stream_ordering); - - --- --- Name: destinations destinations_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY destinations - ADD CONSTRAINT destinations_pkey PRIMARY KEY (destination); - - --- --- Name: devices device_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY devices - ADD CONSTRAINT device_uniqueness UNIQUE (user_id, device_id); - - --- --- Name: e2e_device_keys_json e2e_device_keys_json_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY e2e_device_keys_json - ADD CONSTRAINT e2e_device_keys_json_uniqueness UNIQUE (user_id, device_id); - - --- --- Name: e2e_one_time_keys_json e2e_one_time_keys_json_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY e2e_one_time_keys_json - ADD CONSTRAINT e2e_one_time_keys_json_uniqueness UNIQUE (user_id, device_id, algorithm, key_id); - - --- --- Name: event_backward_extremities event_backward_extremities_event_id_room_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY event_backward_extremities - ADD CONSTRAINT event_backward_extremities_event_id_room_id_key UNIQUE (event_id, room_id); - - --- --- Name: event_content_hashes event_content_hashes_event_id_algorithm_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY event_content_hashes - ADD CONSTRAINT event_content_hashes_event_id_algorithm_key UNIQUE (event_id, algorithm); - - --- --- Name: event_destinations event_destinations_event_id_destination_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY event_destinations - ADD CONSTRAINT event_destinations_event_id_destination_key UNIQUE (event_id, destination); - - --- --- Name: event_edge_hashes event_edge_hashes_event_id_prev_event_id_algorithm_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY event_edge_hashes - ADD CONSTRAINT event_edge_hashes_event_id_prev_event_id_algorithm_key UNIQUE (event_id, prev_event_id, algorithm); - - --- --- Name: event_edges event_edges_event_id_prev_event_id_room_id_is_state_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY event_edges - ADD CONSTRAINT event_edges_event_id_prev_event_id_room_id_is_state_key UNIQUE (event_id, prev_event_id, room_id, is_state); - - --- --- Name: event_forward_extremities event_forward_extremities_event_id_room_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY event_forward_extremities - ADD CONSTRAINT event_forward_extremities_event_id_room_id_key UNIQUE (event_id, room_id); - - --- --- Name: event_push_actions event_id_user_id_profile_tag_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY event_push_actions - ADD CONSTRAINT event_id_user_id_profile_tag_uniqueness UNIQUE (room_id, event_id, user_id, profile_tag); - - --- --- Name: event_json event_json_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY event_json - ADD CONSTRAINT event_json_event_id_key UNIQUE (event_id); - - --- --- Name: event_push_summary_stream_ordering event_push_summary_stream_ordering_lock_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY event_push_summary_stream_ordering - ADD CONSTRAINT event_push_summary_stream_ordering_lock_key UNIQUE (lock); - - --- --- Name: event_reference_hashes event_reference_hashes_event_id_algorithm_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY event_reference_hashes - ADD CONSTRAINT event_reference_hashes_event_id_algorithm_key UNIQUE (event_id, algorithm); - - --- --- Name: event_reports event_reports_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY event_reports - ADD CONSTRAINT event_reports_pkey PRIMARY KEY (id); - - --- --- Name: event_signatures event_signatures_event_id_signature_name_key_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY event_signatures - ADD CONSTRAINT event_signatures_event_id_signature_name_key_id_key UNIQUE (event_id, signature_name, key_id); - - --- --- Name: event_to_state_groups event_to_state_groups_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY event_to_state_groups - ADD CONSTRAINT event_to_state_groups_event_id_key UNIQUE (event_id); - - --- --- Name: events events_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY events - ADD CONSTRAINT events_event_id_key UNIQUE (event_id); - - --- --- Name: events events_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY events - ADD CONSTRAINT events_pkey PRIMARY KEY (stream_ordering); - - --- --- Name: ex_outlier_stream ex_outlier_stream_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY ex_outlier_stream - ADD CONSTRAINT ex_outlier_stream_pkey PRIMARY KEY (event_stream_ordering); - - --- --- Name: feedback feedback_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY feedback - ADD CONSTRAINT feedback_event_id_key UNIQUE (event_id); - - --- --- Name: group_roles group_roles_group_id_role_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY group_roles - ADD CONSTRAINT group_roles_group_id_role_id_key UNIQUE (group_id, role_id); - - --- --- Name: group_room_categories group_room_categories_group_id_category_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY group_room_categories - ADD CONSTRAINT group_room_categories_group_id_category_id_key UNIQUE (group_id, category_id); - - --- --- Name: group_summary_roles group_summary_roles_group_id_role_id_role_order_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY group_summary_roles - ADD CONSTRAINT group_summary_roles_group_id_role_id_role_order_key UNIQUE (group_id, role_id, role_order); - - --- --- Name: group_summary_room_categories group_summary_room_categories_group_id_category_id_cat_orde_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY group_summary_room_categories - ADD CONSTRAINT group_summary_room_categories_group_id_category_id_cat_orde_key UNIQUE (group_id, category_id, cat_order); - - --- --- Name: group_summary_rooms group_summary_rooms_group_id_category_id_room_id_room_order_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY group_summary_rooms - ADD CONSTRAINT group_summary_rooms_group_id_category_id_room_id_room_order_key UNIQUE (group_id, category_id, room_id, room_order); - - --- --- Name: guest_access guest_access_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY guest_access - ADD CONSTRAINT guest_access_event_id_key UNIQUE (event_id); - - --- --- Name: history_visibility history_visibility_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY history_visibility - ADD CONSTRAINT history_visibility_event_id_key UNIQUE (event_id); - - --- --- Name: local_media_repository local_media_repository_media_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY local_media_repository - ADD CONSTRAINT local_media_repository_media_id_key UNIQUE (media_id); - - --- --- Name: local_media_repository_thumbnails local_media_repository_thumbn_media_id_thumbnail_width_thum_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY local_media_repository_thumbnails - ADD CONSTRAINT local_media_repository_thumbn_media_id_thumbnail_width_thum_key UNIQUE (media_id, thumbnail_width, thumbnail_height, thumbnail_type); - - --- --- Name: user_threepids medium_address; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY user_threepids - ADD CONSTRAINT medium_address UNIQUE (medium, address); - - --- --- Name: open_id_tokens open_id_tokens_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY open_id_tokens - ADD CONSTRAINT open_id_tokens_pkey PRIMARY KEY (token); - - --- --- Name: presence_allow_inbound presence_allow_inbound_observed_user_id_observer_user_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY presence_allow_inbound - ADD CONSTRAINT presence_allow_inbound_observed_user_id_observer_user_id_key UNIQUE (observed_user_id, observer_user_id); - - --- --- Name: presence_list presence_list_user_id_observed_user_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY presence_list - ADD CONSTRAINT presence_list_user_id_observed_user_id_key UNIQUE (user_id, observed_user_id); - - --- --- Name: presence presence_user_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY presence - ADD CONSTRAINT presence_user_id_key UNIQUE (user_id); - - --- --- Name: account_data_max_stream_id private_user_data_max_stream_id_lock_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY account_data_max_stream_id - ADD CONSTRAINT private_user_data_max_stream_id_lock_key UNIQUE (lock); - - --- --- Name: profiles profiles_user_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY profiles - ADD CONSTRAINT profiles_user_id_key UNIQUE (user_id); - - --- --- Name: push_rules_enable push_rules_enable_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY push_rules_enable - ADD CONSTRAINT push_rules_enable_pkey PRIMARY KEY (id); - - --- --- Name: push_rules_enable push_rules_enable_user_name_rule_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY push_rules_enable - ADD CONSTRAINT push_rules_enable_user_name_rule_id_key UNIQUE (user_name, rule_id); - - --- --- Name: push_rules push_rules_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY push_rules - ADD CONSTRAINT push_rules_pkey PRIMARY KEY (id); - - --- --- Name: push_rules push_rules_user_name_rule_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY push_rules - ADD CONSTRAINT push_rules_user_name_rule_id_key UNIQUE (user_name, rule_id); - - --- --- Name: pusher_throttle pusher_throttle_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY pusher_throttle - ADD CONSTRAINT pusher_throttle_pkey PRIMARY KEY (pusher, room_id); - - --- --- Name: pushers pushers2_app_id_pushkey_user_name_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY pushers - ADD CONSTRAINT pushers2_app_id_pushkey_user_name_key UNIQUE (app_id, pushkey, user_name); - - --- --- Name: pushers pushers2_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY pushers - ADD CONSTRAINT pushers2_pkey PRIMARY KEY (id); - - --- --- Name: receipts_graph receipts_graph_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY receipts_graph - ADD CONSTRAINT receipts_graph_uniqueness UNIQUE (room_id, receipt_type, user_id); - - --- --- Name: receipts_linearized receipts_linearized_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY receipts_linearized - ADD CONSTRAINT receipts_linearized_uniqueness UNIQUE (room_id, receipt_type, user_id); - - --- --- Name: received_transactions received_transactions_transaction_id_origin_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY received_transactions - ADD CONSTRAINT received_transactions_transaction_id_origin_key UNIQUE (transaction_id, origin); - - --- --- Name: redactions redactions_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY redactions - ADD CONSTRAINT redactions_event_id_key UNIQUE (event_id); - - --- --- Name: rejections rejections_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY rejections - ADD CONSTRAINT rejections_event_id_key UNIQUE (event_id); - - --- --- Name: remote_media_cache remote_media_cache_media_origin_media_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY remote_media_cache - ADD CONSTRAINT remote_media_cache_media_origin_media_id_key UNIQUE (media_origin, media_id); - - --- --- Name: remote_media_cache_thumbnails remote_media_cache_thumbnails_media_origin_media_id_thumbna_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY remote_media_cache_thumbnails - ADD CONSTRAINT remote_media_cache_thumbnails_media_origin_media_id_thumbna_key UNIQUE (media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type); - - --- --- Name: room_account_data room_account_data_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY room_account_data - ADD CONSTRAINT room_account_data_uniqueness UNIQUE (user_id, room_id, account_data_type); - - --- --- Name: room_aliases room_aliases_room_alias_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY room_aliases - ADD CONSTRAINT room_aliases_room_alias_key UNIQUE (room_alias); - - --- --- Name: room_depth room_depth_room_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY room_depth - ADD CONSTRAINT room_depth_room_id_key UNIQUE (room_id); - - --- --- Name: room_hosts room_hosts_room_id_host_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY room_hosts - ADD CONSTRAINT room_hosts_room_id_host_key UNIQUE (room_id, host); - - --- --- Name: room_memberships room_memberships_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY room_memberships - ADD CONSTRAINT room_memberships_event_id_key UNIQUE (event_id); - - --- --- Name: room_names room_names_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY room_names - ADD CONSTRAINT room_names_event_id_key UNIQUE (event_id); - - --- --- Name: room_tags_revisions room_tag_revisions_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY room_tags_revisions - ADD CONSTRAINT room_tag_revisions_uniqueness UNIQUE (user_id, room_id); - - --- --- Name: room_tags room_tag_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY room_tags - ADD CONSTRAINT room_tag_uniqueness UNIQUE (user_id, room_id, tag); - - --- --- Name: rooms rooms_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY rooms - ADD CONSTRAINT rooms_pkey PRIMARY KEY (room_id); - - --- --- Name: server_keys_json server_keys_json_uniqueness; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY server_keys_json - ADD CONSTRAINT server_keys_json_uniqueness UNIQUE (server_name, key_id, from_server); - - --- --- Name: server_signature_keys server_signature_keys_server_name_key_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY server_signature_keys - ADD CONSTRAINT server_signature_keys_server_name_key_id_key UNIQUE (server_name, key_id); - - --- --- Name: server_tls_certificates server_tls_certificates_server_name_fingerprint_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY server_tls_certificates - ADD CONSTRAINT server_tls_certificates_server_name_fingerprint_key UNIQUE (server_name, fingerprint); - - --- --- Name: state_events state_events_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY state_events - ADD CONSTRAINT state_events_event_id_key UNIQUE (event_id); - - --- --- Name: state_forward_extremities state_forward_extremities_event_id_room_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY state_forward_extremities - ADD CONSTRAINT state_forward_extremities_event_id_room_id_key UNIQUE (event_id, room_id); - - --- --- Name: state_groups state_groups_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY state_groups - ADD CONSTRAINT state_groups_pkey PRIMARY KEY (id); - - --- --- Name: topics topics_event_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY topics - ADD CONSTRAINT topics_event_id_key UNIQUE (event_id); - - --- --- Name: transaction_id_to_pdu transaction_id_to_pdu_transaction_id_destination_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY transaction_id_to_pdu - ADD CONSTRAINT transaction_id_to_pdu_transaction_id_destination_key UNIQUE (transaction_id, destination); - - --- --- Name: user_directory_stream_pos user_directory_stream_pos_lock_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY user_directory_stream_pos - ADD CONSTRAINT user_directory_stream_pos_lock_key UNIQUE (lock); - - --- --- Name: users users_name_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY users - ADD CONSTRAINT users_name_key UNIQUE (name); - - --- --- Name: access_tokens_device_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX access_tokens_device_id ON access_tokens USING btree (user_id, device_id); - - --- --- Name: account_data_stream_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX account_data_stream_id ON account_data USING btree (user_id, stream_id); - - --- --- Name: application_services_txns_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX application_services_txns_id ON application_services_txns USING btree (as_id); - - --- --- Name: appservice_room_list_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX appservice_room_list_idx ON appservice_room_list USING btree (appservice_id, network_id, room_id); - - --- --- Name: blocked_rooms_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX blocked_rooms_idx ON blocked_rooms USING btree (room_id); - - --- --- Name: cache_invalidation_stream_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX cache_invalidation_stream_id ON cache_invalidation_stream USING btree (stream_id); - - --- --- Name: current_state_delta_stream_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX current_state_delta_stream_idx ON current_state_delta_stream USING btree (stream_id); - - --- --- Name: current_state_events_member_index; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX current_state_events_member_index ON current_state_events USING btree (state_key) WHERE (type = 'm.room.member'::text); - - --- --- Name: deleted_pushers_stream_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX deleted_pushers_stream_id ON deleted_pushers USING btree (stream_id); - - --- --- Name: device_federation_inbox_sender_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX device_federation_inbox_sender_id ON device_federation_inbox USING btree (origin, message_id); - - --- --- Name: device_federation_outbox_destination_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX device_federation_outbox_destination_id ON device_federation_outbox USING btree (destination, stream_id); - - --- --- Name: device_federation_outbox_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX device_federation_outbox_id ON device_federation_outbox USING btree (stream_id); - - --- --- Name: device_inbox_stream_id_user_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX device_inbox_stream_id_user_id ON device_inbox USING btree (stream_id, user_id); - - --- --- Name: device_inbox_user_stream_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX device_inbox_user_stream_id ON device_inbox USING btree (user_id, device_id, stream_id); - - --- --- Name: device_lists_outbound_last_success_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX device_lists_outbound_last_success_idx ON device_lists_outbound_last_success USING btree (destination, user_id, stream_id); - - --- --- Name: device_lists_outbound_pokes_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX device_lists_outbound_pokes_id ON device_lists_outbound_pokes USING btree (destination, stream_id); - - --- --- Name: device_lists_outbound_pokes_stream; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX device_lists_outbound_pokes_stream ON device_lists_outbound_pokes USING btree (stream_id); - - --- --- Name: device_lists_outbound_pokes_user; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX device_lists_outbound_pokes_user ON device_lists_outbound_pokes USING btree (destination, user_id); - - --- --- Name: device_lists_remote_cache_unique_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX device_lists_remote_cache_unique_id ON device_lists_remote_cache USING btree (user_id, device_id); - - --- --- Name: device_lists_remote_extremeties_unique_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX device_lists_remote_extremeties_unique_idx ON device_lists_remote_extremeties USING btree (user_id); - - --- --- Name: device_lists_stream_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX device_lists_stream_id ON device_lists_stream USING btree (stream_id, user_id); - - --- --- Name: device_lists_stream_user_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX device_lists_stream_user_id ON device_lists_stream USING btree (user_id, device_id); - - --- --- Name: e2e_room_keys_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX e2e_room_keys_idx ON e2e_room_keys USING btree (user_id, room_id, session_id); - - --- --- Name: e2e_room_keys_versions_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions USING btree (user_id, version); - - --- --- Name: erased_users_user; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX erased_users_user ON erased_users USING btree (user_id); - - --- --- Name: ev_b_extrem_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX ev_b_extrem_id ON event_backward_extremities USING btree (event_id); - - --- --- Name: ev_b_extrem_room; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX ev_b_extrem_room ON event_backward_extremities USING btree (room_id); - - --- --- Name: ev_edges_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX ev_edges_id ON event_edges USING btree (event_id); - - --- --- Name: ev_edges_prev_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX ev_edges_prev_id ON event_edges USING btree (prev_event_id); - - --- --- Name: ev_extrem_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX ev_extrem_id ON event_forward_extremities USING btree (event_id); - - --- --- Name: ev_extrem_room; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX ev_extrem_room ON event_forward_extremities USING btree (room_id); - - --- --- Name: evauth_edges_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX evauth_edges_id ON event_auth USING btree (event_id); - - --- --- Name: event_contains_url_index; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX event_contains_url_index ON events USING btree (room_id, topological_ordering, stream_ordering) WHERE ((contains_url = true) AND (outlier = false)); - - --- --- Name: event_json_room_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX event_json_room_id ON event_json USING btree (room_id); - - --- --- Name: event_push_actions_highlights_index; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX event_push_actions_highlights_index ON event_push_actions USING btree (user_id, room_id, topological_ordering, stream_ordering) WHERE (highlight = 1); - - --- --- Name: event_push_actions_rm_tokens; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX event_push_actions_rm_tokens ON event_push_actions USING btree (user_id, room_id, topological_ordering, stream_ordering); - - --- --- Name: event_push_actions_room_id_user_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX event_push_actions_room_id_user_id ON event_push_actions USING btree (room_id, user_id); - - --- --- Name: event_push_actions_staging_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX event_push_actions_staging_id ON event_push_actions_staging USING btree (event_id); - - --- --- Name: event_push_actions_stream_ordering; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX event_push_actions_stream_ordering ON event_push_actions USING btree (stream_ordering, user_id); - - --- --- Name: event_push_actions_u_highlight; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX event_push_actions_u_highlight ON event_push_actions USING btree (user_id, stream_ordering); - - --- --- Name: event_push_summary_user_rm; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX event_push_summary_user_rm ON event_push_summary USING btree (user_id, room_id); - - --- --- Name: event_reference_hashes_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX event_reference_hashes_id ON event_reference_hashes USING btree (event_id); - - --- --- Name: event_search_ev_ridx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX event_search_ev_ridx ON event_search USING btree (room_id); - - --- --- Name: event_search_event_id_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX event_search_event_id_idx ON event_search USING btree (event_id); - - --- --- Name: event_search_fts_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX event_search_fts_idx ON event_search USING gin (vector); - - --- --- Name: event_to_state_groups_sg_index; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX event_to_state_groups_sg_index ON event_to_state_groups USING btree (state_group); - - --- --- Name: events_order_room; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX events_order_room ON events USING btree (room_id, topological_ordering, stream_ordering); - - --- --- Name: events_room_stream; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX events_room_stream ON events USING btree (room_id, stream_ordering); - - --- --- Name: events_ts; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX events_ts ON events USING btree (origin_server_ts, stream_ordering); - - --- --- Name: group_attestations_remote_g_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX group_attestations_remote_g_idx ON group_attestations_remote USING btree (group_id, user_id); - - --- --- Name: group_attestations_remote_u_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX group_attestations_remote_u_idx ON group_attestations_remote USING btree (user_id); - - --- --- Name: group_attestations_remote_v_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX group_attestations_remote_v_idx ON group_attestations_remote USING btree (valid_until_ms); - - --- --- Name: group_attestations_renewals_g_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX group_attestations_renewals_g_idx ON group_attestations_renewals USING btree (group_id, user_id); - - --- --- Name: group_attestations_renewals_u_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX group_attestations_renewals_u_idx ON group_attestations_renewals USING btree (user_id); - - --- --- Name: group_attestations_renewals_v_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX group_attestations_renewals_v_idx ON group_attestations_renewals USING btree (valid_until_ms); - - --- --- Name: group_invites_g_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX group_invites_g_idx ON group_invites USING btree (group_id, user_id); - - --- --- Name: group_invites_u_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX group_invites_u_idx ON group_invites USING btree (user_id); - - --- --- Name: group_rooms_g_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX group_rooms_g_idx ON group_rooms USING btree (group_id, room_id); - - --- --- Name: group_rooms_r_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX group_rooms_r_idx ON group_rooms USING btree (room_id); - - --- --- Name: group_summary_rooms_g_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX group_summary_rooms_g_idx ON group_summary_rooms USING btree (group_id, room_id, category_id); - - --- --- Name: group_summary_users_g_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX group_summary_users_g_idx ON group_summary_users USING btree (group_id); - - --- --- Name: group_users_g_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX group_users_g_idx ON group_users USING btree (group_id, user_id); - - --- --- Name: group_users_u_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX group_users_u_idx ON group_users USING btree (user_id); - - --- --- Name: groups_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX groups_idx ON groups USING btree (group_id); - - --- --- Name: local_group_membership_g_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX local_group_membership_g_idx ON local_group_membership USING btree (group_id); - - --- --- Name: local_group_membership_u_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX local_group_membership_u_idx ON local_group_membership USING btree (user_id, group_id); - - --- --- Name: local_invites_for_user_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX local_invites_for_user_idx ON local_invites USING btree (invitee, locally_rejected, replaced_by, room_id); - - --- --- Name: local_invites_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX local_invites_id ON local_invites USING btree (stream_id); - - --- --- Name: local_media_repository_thumbnails_media_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails USING btree (media_id); - - --- --- Name: local_media_repository_url_cache_by_url_download_ts; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX local_media_repository_url_cache_by_url_download_ts ON local_media_repository_url_cache USING btree (url, download_ts); - - --- --- Name: local_media_repository_url_cache_expires_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX local_media_repository_url_cache_expires_idx ON local_media_repository_url_cache USING btree (expires_ts); - - --- --- Name: local_media_repository_url_cache_media_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX local_media_repository_url_cache_media_idx ON local_media_repository_url_cache USING btree (media_id); - - --- --- Name: local_media_repository_url_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX local_media_repository_url_idx ON local_media_repository USING btree (created_ts) WHERE (url_cache IS NOT NULL); - - --- --- Name: monthly_active_users_time_stamp; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX monthly_active_users_time_stamp ON monthly_active_users USING btree ("timestamp"); - - --- --- Name: monthly_active_users_users; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX monthly_active_users_users ON monthly_active_users USING btree (user_id); - - --- --- Name: open_id_tokens_ts_valid_until_ms; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX open_id_tokens_ts_valid_until_ms ON open_id_tokens USING btree (ts_valid_until_ms); - - --- --- Name: presence_list_user_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX presence_list_user_id ON presence_list USING btree (user_id); - - --- --- Name: presence_stream_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX presence_stream_id ON presence_stream USING btree (stream_id, user_id); - - --- --- Name: presence_stream_user_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX presence_stream_user_id ON presence_stream USING btree (user_id); - - --- --- Name: public_room_index; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX public_room_index ON rooms USING btree (is_public); - - --- --- Name: public_room_list_stream_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX public_room_list_stream_idx ON public_room_list_stream USING btree (stream_id); - - --- --- Name: public_room_list_stream_rm_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX public_room_list_stream_rm_idx ON public_room_list_stream USING btree (room_id, stream_id); - - --- --- Name: push_rules_enable_user_name; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX push_rules_enable_user_name ON push_rules_enable USING btree (user_name); - - --- --- Name: push_rules_stream_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX push_rules_stream_id ON push_rules_stream USING btree (stream_id); - - --- --- Name: push_rules_stream_user_stream_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX push_rules_stream_user_stream_id ON push_rules_stream USING btree (user_id, stream_id); - - --- --- Name: push_rules_user_name; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX push_rules_user_name ON push_rules USING btree (user_name); - - --- --- Name: ratelimit_override_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX ratelimit_override_idx ON ratelimit_override USING btree (user_id); - - --- --- Name: receipts_linearized_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX receipts_linearized_id ON receipts_linearized USING btree (stream_id); - - --- --- Name: receipts_linearized_room_stream; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX receipts_linearized_room_stream ON receipts_linearized USING btree (room_id, stream_id); - - --- --- Name: receipts_linearized_user; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX receipts_linearized_user ON receipts_linearized USING btree (user_id); - - --- --- Name: received_transactions_ts; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX received_transactions_ts ON received_transactions USING btree (ts); - - --- --- Name: redactions_redacts; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX redactions_redacts ON redactions USING btree (redacts); - - --- --- Name: remote_profile_cache_time; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX remote_profile_cache_time ON remote_profile_cache USING btree (last_check); - - --- --- Name: remote_profile_cache_user_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX remote_profile_cache_user_id ON remote_profile_cache USING btree (user_id); - - --- --- Name: room_account_data_stream_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX room_account_data_stream_id ON room_account_data USING btree (user_id, stream_id); - - --- --- Name: room_alias_servers_alias; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX room_alias_servers_alias ON room_alias_servers USING btree (room_alias); - - --- --- Name: room_aliases_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX room_aliases_id ON room_aliases USING btree (room_id); - - --- --- Name: room_depth_room; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX room_depth_room ON room_depth USING btree (room_id); - - --- --- Name: room_memberships_room_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX room_memberships_room_id ON room_memberships USING btree (room_id); - - --- --- Name: room_memberships_user_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX room_memberships_user_id ON room_memberships USING btree (user_id); - - --- --- Name: room_names_room_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX room_names_room_id ON room_names USING btree (room_id); - - --- --- Name: st_extrem_keys; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX st_extrem_keys ON state_forward_extremities USING btree (room_id, type, state_key); - - --- --- Name: state_group_edges_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX state_group_edges_idx ON state_group_edges USING btree (state_group); - - --- --- Name: state_group_edges_prev_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX state_group_edges_prev_idx ON state_group_edges USING btree (prev_state_group); - - --- --- Name: state_groups_state_type_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX state_groups_state_type_idx ON state_groups_state USING btree (state_group, type, state_key); - - --- --- Name: stream_ordering_to_exterm_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX stream_ordering_to_exterm_idx ON stream_ordering_to_exterm USING btree (stream_ordering); - - --- --- Name: stream_ordering_to_exterm_rm_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX stream_ordering_to_exterm_rm_idx ON stream_ordering_to_exterm USING btree (room_id, stream_ordering); - - --- --- Name: threepid_guest_access_tokens_index; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX threepid_guest_access_tokens_index ON threepid_guest_access_tokens USING btree (medium, address); - - --- --- Name: topics_room_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX topics_room_id ON topics USING btree (room_id); - - --- --- Name: transaction_id_to_pdu_dest; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu USING btree (destination); - - --- --- Name: user_daily_visits_ts_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX user_daily_visits_ts_idx ON user_daily_visits USING btree ("timestamp"); - - --- --- Name: user_daily_visits_uts_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX user_daily_visits_uts_idx ON user_daily_visits USING btree (user_id, "timestamp"); - - --- --- Name: user_directory_room_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX user_directory_room_idx ON user_directory USING btree (room_id); - - --- --- Name: user_directory_search_fts_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX user_directory_search_fts_idx ON user_directory_search USING gin (vector); - - --- --- Name: user_directory_search_user_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX user_directory_search_user_idx ON user_directory_search USING btree (user_id); - - --- --- Name: user_directory_user_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX user_directory_user_idx ON user_directory USING btree (user_id); - - --- --- Name: user_filters_by_user_id_filter_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX user_filters_by_user_id_filter_id ON user_filters USING btree (user_id, filter_id); - - --- --- Name: user_ips_device_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX user_ips_device_id ON user_ips USING btree (user_id, device_id, last_seen); - - --- --- Name: user_ips_device_unique_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX user_ips_device_unique_id ON user_ips USING btree (user_id, access_token, ip, user_agent, device_id); - - --- --- Name: user_ips_last_seen; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX user_ips_last_seen ON user_ips USING btree (user_id, last_seen); - - --- --- Name: user_ips_last_seen_only; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX user_ips_last_seen_only ON user_ips USING btree (last_seen); - - --- --- Name: user_ips_user_ip; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX user_ips_user_ip ON user_ips USING btree (user_id, access_token, ip); - - --- --- Name: user_threepids_medium_address; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX user_threepids_medium_address ON user_threepids USING btree (medium, address); - - --- --- Name: user_threepids_user_id; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX user_threepids_user_id ON user_threepids USING btree (user_id); - - --- --- Name: users_creation_ts; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX users_creation_ts ON users USING btree (creation_ts); - - --- --- Name: users_in_public_rooms_room_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX users_in_public_rooms_room_idx ON users_in_public_rooms USING btree (room_id); - - --- --- Name: users_in_public_rooms_user_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX users_in_public_rooms_user_idx ON users_in_public_rooms USING btree (user_id); - - --- --- Name: users_who_share_rooms_o_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX users_who_share_rooms_o_idx ON users_who_share_rooms USING btree (other_user_id); - - --- --- Name: users_who_share_rooms_r_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX users_who_share_rooms_r_idx ON users_who_share_rooms USING btree (room_id); - - --- --- Name: users_who_share_rooms_u_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX users_who_share_rooms_u_idx ON users_who_share_rooms USING btree (user_id, other_user_id); - - --- --- Name: application_services_regex application_services_regex_as_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY application_services_regex - ADD CONSTRAINT application_services_regex_as_id_fkey FOREIGN KEY (as_id) REFERENCES application_services(id); - - --- --- PostgreSQL database dump complete --- - diff --git a/synapse/storage/schema/full_schemas/54/full.sql.sqlite b/synapse/storage/schema/full_schemas/54/full.sql.sqlite deleted file mode 100644 index 60b8c6fe7629..000000000000 --- a/synapse/storage/schema/full_schemas/54/full.sql.sqlite +++ /dev/null @@ -1,1203 +0,0 @@ -CREATE TABLE application_services( - id BIGINT PRIMARY KEY, - url TEXT, - token TEXT, - hs_token TEXT, - sender TEXT, - UNIQUE(token) -); -CREATE TABLE application_services_regex( - id BIGINT PRIMARY KEY, - as_id BIGINT NOT NULL, - namespace INTEGER, - regex TEXT, - FOREIGN KEY(as_id) REFERENCES application_services(id) -); -CREATE TABLE application_services_state( - as_id TEXT PRIMARY KEY, - state VARCHAR(5), - last_txn INTEGER -); -CREATE TABLE application_services_txns( - as_id TEXT NOT NULL, - txn_id INTEGER NOT NULL, - event_ids TEXT NOT NULL, - UNIQUE(as_id, txn_id) -); -CREATE INDEX application_services_txns_id ON application_services_txns(as_id); -CREATE TABLE presence( - user_id TEXT NOT NULL, - state VARCHAR(20), - status_msg TEXT, - mtime BIGINT, - UNIQUE(user_id) -); -CREATE TABLE presence_allow_inbound( - observed_user_id TEXT NOT NULL, - observer_user_id TEXT NOT NULL, - UNIQUE(observed_user_id, observer_user_id) -); -CREATE TABLE presence_list( - user_id TEXT NOT NULL, - observed_user_id TEXT NOT NULL, - accepted BOOLEAN NOT NULL, - UNIQUE(user_id, observed_user_id) -); -CREATE INDEX presence_list_user_id ON presence_list(user_id); -CREATE TABLE users( - name TEXT, - password_hash TEXT, - creation_ts BIGINT, - admin SMALLINT DEFAULT 0 NOT NULL, - upgrade_ts BIGINT, - is_guest SMALLINT DEFAULT 0 NOT NULL, - appservice_id TEXT, - consent_version TEXT, - consent_server_notice_sent TEXT, - user_type TEXT DEFAULT NULL, - UNIQUE(name) -); -CREATE TABLE access_tokens( - id BIGINT PRIMARY KEY, - user_id TEXT NOT NULL, - device_id TEXT, - token TEXT NOT NULL, - last_used BIGINT, - UNIQUE(token) -); -CREATE TABLE user_ips( - user_id TEXT NOT NULL, - access_token TEXT NOT NULL, - device_id TEXT, - ip TEXT NOT NULL, - user_agent TEXT NOT NULL, - last_seen BIGINT NOT NULL -); -CREATE INDEX user_ips_user_ip ON user_ips(user_id, access_token, ip); -CREATE TABLE profiles( - user_id TEXT NOT NULL, - displayname TEXT, - avatar_url TEXT, - UNIQUE(user_id) -); -CREATE TABLE received_transactions( - transaction_id TEXT, - origin TEXT, - ts BIGINT, - response_code INTEGER, - response_json bytea, - has_been_referenced smallint default 0, - UNIQUE(transaction_id, origin) -); -CREATE TABLE transaction_id_to_pdu( - transaction_id INTEGER, - destination TEXT, - pdu_id TEXT, - pdu_origin TEXT, - UNIQUE(transaction_id, destination) -); -CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination); -CREATE TABLE destinations( - destination TEXT PRIMARY KEY, - retry_last_ts BIGINT, - retry_interval INTEGER -); -CREATE TABLE events( - stream_ordering INTEGER PRIMARY KEY, - topological_ordering BIGINT NOT NULL, - event_id TEXT NOT NULL, - type TEXT NOT NULL, - room_id TEXT NOT NULL, - content TEXT, - unrecognized_keys TEXT, - processed BOOL NOT NULL, - outlier BOOL NOT NULL, - depth BIGINT DEFAULT 0 NOT NULL, - origin_server_ts BIGINT, - received_ts BIGINT, - sender TEXT, - contains_url BOOLEAN, - UNIQUE(event_id) -); -CREATE INDEX events_order_room ON events( - room_id, - topological_ordering, - stream_ordering -); -CREATE TABLE event_json( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - internal_metadata TEXT NOT NULL, - json TEXT NOT NULL, - UNIQUE(event_id) -); -CREATE INDEX event_json_room_id ON event_json(room_id); -CREATE TABLE state_events( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - prev_state TEXT, - UNIQUE(event_id) -); -CREATE TABLE current_state_events( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - UNIQUE(event_id), - UNIQUE(room_id, type, state_key) -); -CREATE TABLE room_memberships( - event_id TEXT NOT NULL, - user_id TEXT NOT NULL, - sender TEXT NOT NULL, - room_id TEXT NOT NULL, - membership TEXT NOT NULL, - forgotten INTEGER DEFAULT 0, - display_name TEXT, - avatar_url TEXT, - UNIQUE(event_id) -); -CREATE INDEX room_memberships_room_id ON room_memberships(room_id); -CREATE INDEX room_memberships_user_id ON room_memberships(user_id); -CREATE TABLE feedback( - event_id TEXT NOT NULL, - feedback_type TEXT, - target_event_id TEXT, - sender TEXT, - room_id TEXT, - UNIQUE(event_id) -); -CREATE TABLE topics( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - topic TEXT NOT NULL, - UNIQUE(event_id) -); -CREATE INDEX topics_room_id ON topics(room_id); -CREATE TABLE room_names( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - name TEXT NOT NULL, - UNIQUE(event_id) -); -CREATE INDEX room_names_room_id ON room_names(room_id); -CREATE TABLE rooms( - room_id TEXT PRIMARY KEY NOT NULL, - is_public BOOL, - creator TEXT -); -CREATE TABLE room_hosts( - room_id TEXT NOT NULL, - host TEXT NOT NULL, - UNIQUE(room_id, host) -); -CREATE TABLE server_tls_certificates( - server_name TEXT, - fingerprint TEXT, - from_server TEXT, - ts_added_ms BIGINT, - tls_certificate bytea, - UNIQUE(server_name, fingerprint) -); -CREATE TABLE server_signature_keys( - server_name TEXT, - key_id TEXT, - from_server TEXT, - ts_added_ms BIGINT, - verify_key bytea, - UNIQUE(server_name, key_id) -); -CREATE TABLE rejections( - event_id TEXT NOT NULL, - reason TEXT NOT NULL, - last_check TEXT NOT NULL, - UNIQUE(event_id) -); -CREATE TABLE push_rules( - id BIGINT PRIMARY KEY, - user_name TEXT NOT NULL, - rule_id TEXT NOT NULL, - priority_class SMALLINT NOT NULL, - priority INTEGER NOT NULL DEFAULT 0, - conditions TEXT NOT NULL, - actions TEXT NOT NULL, - UNIQUE(user_name, rule_id) -); -CREATE INDEX push_rules_user_name on push_rules(user_name); -CREATE TABLE user_filters(user_id TEXT, filter_id BIGINT, filter_json bytea); -CREATE INDEX user_filters_by_user_id_filter_id ON user_filters( - user_id, - filter_id -); -CREATE TABLE push_rules_enable( - id BIGINT PRIMARY KEY, - user_name TEXT NOT NULL, - rule_id TEXT NOT NULL, - enabled SMALLINT, - UNIQUE(user_name, rule_id) -); -CREATE INDEX push_rules_enable_user_name on push_rules_enable(user_name); -CREATE TABLE event_forward_extremities( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - UNIQUE(event_id, room_id) -); -CREATE INDEX ev_extrem_room ON event_forward_extremities(room_id); -CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id); -CREATE TABLE event_backward_extremities( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - UNIQUE(event_id, room_id) -); -CREATE INDEX ev_b_extrem_room ON event_backward_extremities(room_id); -CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id); -CREATE TABLE event_edges( - event_id TEXT NOT NULL, - prev_event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - is_state BOOL NOT NULL, - UNIQUE(event_id, prev_event_id, room_id, is_state) -); -CREATE INDEX ev_edges_id ON event_edges(event_id); -CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id); -CREATE TABLE room_depth( - room_id TEXT NOT NULL, - min_depth INTEGER NOT NULL, - UNIQUE(room_id) -); -CREATE INDEX room_depth_room ON room_depth(room_id); -CREATE TABLE event_destinations( - event_id TEXT NOT NULL, - destination TEXT NOT NULL, - delivered_ts BIGINT DEFAULT 0, - UNIQUE(event_id, destination) -); -CREATE TABLE state_forward_extremities( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - UNIQUE(event_id, room_id) -); -CREATE INDEX st_extrem_keys ON state_forward_extremities( - room_id, - type, - state_key -); -CREATE TABLE state_groups( - id BIGINT PRIMARY KEY, - room_id TEXT NOT NULL, - event_id TEXT NOT NULL -); -CREATE TABLE state_groups_state( - state_group BIGINT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - event_id TEXT NOT NULL -); -CREATE TABLE event_to_state_groups( - event_id TEXT NOT NULL, - state_group BIGINT NOT NULL, - UNIQUE(event_id) -); -CREATE TABLE local_media_repository( - media_id TEXT, - media_type TEXT, - media_length INTEGER, - created_ts BIGINT, - upload_name TEXT, - user_id TEXT, - quarantined_by TEXT, - url_cache TEXT, - last_access_ts BIGINT, - UNIQUE(media_id) -); -CREATE TABLE local_media_repository_thumbnails( - media_id TEXT, - thumbnail_width INTEGER, - thumbnail_height INTEGER, - thumbnail_type TEXT, - thumbnail_method TEXT, - thumbnail_length INTEGER, - UNIQUE(media_id, thumbnail_width, thumbnail_height, thumbnail_type) -); -CREATE INDEX local_media_repository_thumbnails_media_id ON local_media_repository_thumbnails( - media_id -); -CREATE TABLE remote_media_cache( - media_origin TEXT, - media_id TEXT, - media_type TEXT, - created_ts BIGINT, - upload_name TEXT, - media_length INTEGER, - filesystem_id TEXT, - last_access_ts BIGINT, - quarantined_by TEXT, - UNIQUE(media_origin, media_id) -); -CREATE TABLE remote_media_cache_thumbnails( - media_origin TEXT, - media_id TEXT, - thumbnail_width INTEGER, - thumbnail_height INTEGER, - thumbnail_method TEXT, - thumbnail_type TEXT, - thumbnail_length INTEGER, - filesystem_id TEXT, - UNIQUE(media_origin, media_id, thumbnail_width, thumbnail_height, thumbnail_type) -); -CREATE TABLE redactions( - event_id TEXT NOT NULL, - redacts TEXT NOT NULL, - UNIQUE(event_id) -); -CREATE INDEX redactions_redacts ON redactions(redacts); -CREATE TABLE room_aliases( - room_alias TEXT NOT NULL, - room_id TEXT NOT NULL, - creator TEXT, - UNIQUE(room_alias) -); -CREATE INDEX room_aliases_id ON room_aliases(room_id); -CREATE TABLE room_alias_servers( - room_alias TEXT NOT NULL, - server TEXT NOT NULL -); -CREATE INDEX room_alias_servers_alias ON room_alias_servers(room_alias); -CREATE TABLE event_content_hashes( - event_id TEXT, - algorithm TEXT, - hash bytea, - UNIQUE(event_id, algorithm) -); -CREATE TABLE event_reference_hashes( - event_id TEXT, - algorithm TEXT, - hash bytea, - UNIQUE(event_id, algorithm) -); -CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id); -CREATE TABLE event_signatures( - event_id TEXT, - signature_name TEXT, - key_id TEXT, - signature bytea, - UNIQUE(event_id, signature_name, key_id) -); -CREATE TABLE event_edge_hashes( - event_id TEXT, - prev_event_id TEXT, - algorithm TEXT, - hash bytea, - UNIQUE(event_id, prev_event_id, algorithm) -); -CREATE TABLE IF NOT EXISTS "server_keys_json"( - server_name TEXT NOT NULL, - key_id TEXT NOT NULL, - from_server TEXT NOT NULL, - ts_added_ms BIGINT NOT NULL, - ts_valid_until_ms BIGINT NOT NULL, - key_json bytea NOT NULL, - CONSTRAINT server_keys_json_uniqueness UNIQUE(server_name, key_id, from_server) -); -CREATE TABLE e2e_device_keys_json( - user_id TEXT NOT NULL, - device_id TEXT NOT NULL, - ts_added_ms BIGINT NOT NULL, - key_json TEXT NOT NULL, - CONSTRAINT e2e_device_keys_json_uniqueness UNIQUE(user_id, device_id) -); -CREATE TABLE e2e_one_time_keys_json( - user_id TEXT NOT NULL, - device_id TEXT NOT NULL, - algorithm TEXT NOT NULL, - key_id TEXT NOT NULL, - ts_added_ms BIGINT NOT NULL, - key_json TEXT NOT NULL, - CONSTRAINT e2e_one_time_keys_json_uniqueness UNIQUE(user_id, device_id, algorithm, key_id) -); -CREATE TABLE receipts_graph( - room_id TEXT NOT NULL, - receipt_type TEXT NOT NULL, - user_id TEXT NOT NULL, - event_ids TEXT NOT NULL, - data TEXT NOT NULL, - CONSTRAINT receipts_graph_uniqueness UNIQUE(room_id, receipt_type, user_id) -); -CREATE TABLE receipts_linearized( - stream_id BIGINT NOT NULL, - room_id TEXT NOT NULL, - receipt_type TEXT NOT NULL, - user_id TEXT NOT NULL, - event_id TEXT NOT NULL, - data TEXT NOT NULL, - CONSTRAINT receipts_linearized_uniqueness UNIQUE(room_id, receipt_type, user_id) -); -CREATE INDEX receipts_linearized_id ON receipts_linearized(stream_id); -CREATE INDEX receipts_linearized_room_stream ON receipts_linearized( - room_id, - stream_id -); -CREATE TABLE IF NOT EXISTS "user_threepids"( - user_id TEXT NOT NULL, - medium TEXT NOT NULL, - address TEXT NOT NULL, - validated_at BIGINT NOT NULL, - added_at BIGINT NOT NULL, - CONSTRAINT medium_address UNIQUE(medium, address) -); -CREATE INDEX user_threepids_user_id ON user_threepids(user_id); -CREATE TABLE stats_reporting( - reported_stream_token INTEGER, - reported_time BIGINT -); -CREATE TABLE background_updates( - update_name TEXT NOT NULL, - progress_json TEXT NOT NULL, - depends_on TEXT, - CONSTRAINT background_updates_uniqueness UNIQUE(update_name) -); -CREATE VIRTUAL TABLE event_search USING fts4( - event_id, - room_id, - sender, - key, - value -) -/* event_search( - event_id, - room_id, - sender, - "key", - value -) */; -CREATE TABLE IF NOT EXISTS 'event_search_content'( - docid INTEGER PRIMARY KEY, - 'c0event_id', - 'c1room_id', - 'c2sender', - 'c3key', - 'c4value' -); -CREATE TABLE IF NOT EXISTS 'event_search_segments'(blockid INTEGER PRIMARY KEY, block BLOB); -CREATE TABLE IF NOT EXISTS 'event_search_segdir'( - level INTEGER, - idx INTEGER, - start_block INTEGER, - leaves_end_block INTEGER, - end_block INTEGER, - root BLOB, - PRIMARY KEY(level, idx) -); -CREATE TABLE IF NOT EXISTS 'event_search_docsize'(docid INTEGER PRIMARY KEY, size BLOB); -CREATE TABLE IF NOT EXISTS 'event_search_stat'(id INTEGER PRIMARY KEY, value BLOB); -CREATE TABLE guest_access( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - guest_access TEXT NOT NULL, - UNIQUE(event_id) -); -CREATE TABLE history_visibility( - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - history_visibility TEXT NOT NULL, - UNIQUE(event_id) -); -CREATE TABLE room_tags( - user_id TEXT NOT NULL, - room_id TEXT NOT NULL, - tag TEXT NOT NULL, - content TEXT NOT NULL, - CONSTRAINT room_tag_uniqueness UNIQUE(user_id, room_id, tag) -); -CREATE TABLE room_tags_revisions( - user_id TEXT NOT NULL, - room_id TEXT NOT NULL, - stream_id BIGINT NOT NULL, - CONSTRAINT room_tag_revisions_uniqueness UNIQUE(user_id, room_id) -); -CREATE TABLE IF NOT EXISTS "account_data_max_stream_id"( - Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, - stream_id BIGINT NOT NULL, - CHECK(Lock='X') -); -CREATE TABLE account_data( - user_id TEXT NOT NULL, - account_data_type TEXT NOT NULL, - stream_id BIGINT NOT NULL, - content TEXT NOT NULL, - CONSTRAINT account_data_uniqueness UNIQUE(user_id, account_data_type) -); -CREATE TABLE room_account_data( - user_id TEXT NOT NULL, - room_id TEXT NOT NULL, - account_data_type TEXT NOT NULL, - stream_id BIGINT NOT NULL, - content TEXT NOT NULL, - CONSTRAINT room_account_data_uniqueness UNIQUE(user_id, room_id, account_data_type) -); -CREATE INDEX account_data_stream_id on account_data(user_id, stream_id); -CREATE INDEX room_account_data_stream_id on room_account_data( - user_id, - stream_id -); -CREATE INDEX events_ts ON events(origin_server_ts, stream_ordering); -CREATE TABLE event_push_actions( - room_id TEXT NOT NULL, - event_id TEXT NOT NULL, - user_id TEXT NOT NULL, - profile_tag VARCHAR(32), - actions TEXT NOT NULL, - topological_ordering BIGINT, - stream_ordering BIGINT, - notif SMALLINT, - highlight SMALLINT, - CONSTRAINT event_id_user_id_profile_tag_uniqueness UNIQUE(room_id, event_id, user_id, profile_tag) -); -CREATE INDEX event_push_actions_room_id_user_id on event_push_actions( - room_id, - user_id -); -CREATE INDEX events_room_stream on events(room_id, stream_ordering); -CREATE INDEX public_room_index on rooms(is_public); -CREATE INDEX receipts_linearized_user ON receipts_linearized(user_id); -CREATE INDEX event_push_actions_rm_tokens on event_push_actions( - user_id, - room_id, - topological_ordering, - stream_ordering -); -CREATE TABLE presence_stream( - stream_id BIGINT, - user_id TEXT, - state TEXT, - last_active_ts BIGINT, - last_federation_update_ts BIGINT, - last_user_sync_ts BIGINT, - status_msg TEXT, - currently_active BOOLEAN -); -CREATE INDEX presence_stream_id ON presence_stream(stream_id, user_id); -CREATE INDEX presence_stream_user_id ON presence_stream(user_id); -CREATE TABLE push_rules_stream( - stream_id BIGINT NOT NULL, - event_stream_ordering BIGINT NOT NULL, - user_id TEXT NOT NULL, - rule_id TEXT NOT NULL, - op TEXT NOT NULL, - priority_class SMALLINT, - priority INTEGER, - conditions TEXT, - actions TEXT -); -CREATE INDEX push_rules_stream_id ON push_rules_stream(stream_id); -CREATE INDEX push_rules_stream_user_stream_id on push_rules_stream( - user_id, - stream_id -); -CREATE TABLE current_state_resets( - event_stream_ordering BIGINT PRIMARY KEY NOT NULL -); -CREATE TABLE ex_outlier_stream( - event_stream_ordering BIGINT PRIMARY KEY NOT NULL, - event_id TEXT NOT NULL, - state_group BIGINT NOT NULL -); -CREATE TABLE threepid_guest_access_tokens( - medium TEXT, - address TEXT, - guest_access_token TEXT, - first_inviter TEXT -); -CREATE UNIQUE INDEX threepid_guest_access_tokens_index ON threepid_guest_access_tokens( - medium, - address -); -CREATE TABLE local_invites( - stream_id BIGINT NOT NULL, - inviter TEXT NOT NULL, - invitee TEXT NOT NULL, - event_id TEXT NOT NULL, - room_id TEXT NOT NULL, - locally_rejected TEXT, - replaced_by TEXT -); -CREATE INDEX local_invites_id ON local_invites(stream_id); -CREATE INDEX local_invites_for_user_idx ON local_invites( - invitee, - locally_rejected, - replaced_by, - room_id -); -CREATE INDEX event_push_actions_stream_ordering on event_push_actions( - stream_ordering, - user_id -); -CREATE TABLE open_id_tokens( - token TEXT NOT NULL PRIMARY KEY, - ts_valid_until_ms bigint NOT NULL, - user_id TEXT NOT NULL, - UNIQUE(token) -); -CREATE INDEX open_id_tokens_ts_valid_until_ms ON open_id_tokens( - ts_valid_until_ms -); -CREATE TABLE pusher_throttle( - pusher BIGINT NOT NULL, - room_id TEXT NOT NULL, - last_sent_ts BIGINT, - throttle_ms BIGINT, - PRIMARY KEY(pusher, room_id) -); -CREATE TABLE event_reports( - id BIGINT NOT NULL PRIMARY KEY, - received_ts BIGINT NOT NULL, - room_id TEXT NOT NULL, - event_id TEXT NOT NULL, - user_id TEXT NOT NULL, - reason TEXT, - content TEXT -); -CREATE TABLE devices( - user_id TEXT NOT NULL, - device_id TEXT NOT NULL, - display_name TEXT, - CONSTRAINT device_uniqueness UNIQUE(user_id, device_id) -); -CREATE TABLE appservice_stream_position( - Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, - stream_ordering BIGINT, - CHECK(Lock='X') -); -CREATE TABLE device_inbox( - user_id TEXT NOT NULL, - device_id TEXT NOT NULL, - stream_id BIGINT NOT NULL, - message_json TEXT NOT NULL -); -CREATE INDEX device_inbox_user_stream_id ON device_inbox( - user_id, - device_id, - stream_id -); -CREATE INDEX received_transactions_ts ON received_transactions(ts); -CREATE TABLE device_federation_outbox( - destination TEXT NOT NULL, - stream_id BIGINT NOT NULL, - queued_ts BIGINT NOT NULL, - messages_json TEXT NOT NULL -); -CREATE INDEX device_federation_outbox_destination_id ON device_federation_outbox( - destination, - stream_id -); -CREATE TABLE device_federation_inbox( - origin TEXT NOT NULL, - message_id TEXT NOT NULL, - received_ts BIGINT NOT NULL -); -CREATE INDEX device_federation_inbox_sender_id ON device_federation_inbox( - origin, - message_id -); -CREATE TABLE device_max_stream_id(stream_id BIGINT NOT NULL); -CREATE TABLE public_room_list_stream( - stream_id BIGINT NOT NULL, - room_id TEXT NOT NULL, - visibility BOOLEAN NOT NULL , - appservice_id TEXT, - network_id TEXT -); -CREATE INDEX public_room_list_stream_idx on public_room_list_stream(stream_id); -CREATE INDEX public_room_list_stream_rm_idx on public_room_list_stream( - room_id, - stream_id -); -CREATE TABLE state_group_edges( - state_group BIGINT NOT NULL, - prev_state_group BIGINT NOT NULL -); -CREATE INDEX state_group_edges_idx ON state_group_edges(state_group); -CREATE INDEX state_group_edges_prev_idx ON state_group_edges(prev_state_group); -CREATE TABLE stream_ordering_to_exterm( - stream_ordering BIGINT NOT NULL, - room_id TEXT NOT NULL, - event_id TEXT NOT NULL -); -CREATE INDEX stream_ordering_to_exterm_idx on stream_ordering_to_exterm( - stream_ordering -); -CREATE INDEX stream_ordering_to_exterm_rm_idx on stream_ordering_to_exterm( - room_id, - stream_ordering -); -CREATE TABLE IF NOT EXISTS "event_auth"( - event_id TEXT NOT NULL, - auth_id TEXT NOT NULL, - room_id TEXT NOT NULL -); -CREATE INDEX evauth_edges_id ON event_auth(event_id); -CREATE INDEX user_threepids_medium_address on user_threepids(medium, address); -CREATE TABLE appservice_room_list( - appservice_id TEXT NOT NULL, - network_id TEXT NOT NULL, - room_id TEXT NOT NULL -); -CREATE UNIQUE INDEX appservice_room_list_idx ON appservice_room_list( - appservice_id, - network_id, - room_id -); -CREATE INDEX device_federation_outbox_id ON device_federation_outbox( - stream_id -); -CREATE TABLE federation_stream_position( - type TEXT NOT NULL, - stream_id INTEGER NOT NULL -); -CREATE TABLE device_lists_remote_cache( - user_id TEXT NOT NULL, - device_id TEXT NOT NULL, - content TEXT NOT NULL -); -CREATE TABLE device_lists_remote_extremeties( - user_id TEXT NOT NULL, - stream_id TEXT NOT NULL -); -CREATE TABLE device_lists_stream( - stream_id BIGINT NOT NULL, - user_id TEXT NOT NULL, - device_id TEXT NOT NULL -); -CREATE INDEX device_lists_stream_id ON device_lists_stream(stream_id, user_id); -CREATE TABLE device_lists_outbound_pokes( - destination TEXT NOT NULL, - stream_id BIGINT NOT NULL, - user_id TEXT NOT NULL, - device_id TEXT NOT NULL, - sent BOOLEAN NOT NULL, - ts BIGINT NOT NULL -); -CREATE INDEX device_lists_outbound_pokes_id ON device_lists_outbound_pokes( - destination, - stream_id -); -CREATE INDEX device_lists_outbound_pokes_user ON device_lists_outbound_pokes( - destination, - user_id -); -CREATE TABLE event_push_summary( - user_id TEXT NOT NULL, - room_id TEXT NOT NULL, - notif_count BIGINT NOT NULL, - stream_ordering BIGINT NOT NULL -); -CREATE INDEX event_push_summary_user_rm ON event_push_summary( - user_id, - room_id -); -CREATE TABLE event_push_summary_stream_ordering( - Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, - stream_ordering BIGINT NOT NULL, - CHECK(Lock='X') -); -CREATE TABLE IF NOT EXISTS "pushers"( - id BIGINT PRIMARY KEY, - user_name TEXT NOT NULL, - access_token BIGINT DEFAULT NULL, - profile_tag TEXT NOT NULL, - kind TEXT NOT NULL, - app_id TEXT NOT NULL, - app_display_name TEXT NOT NULL, - device_display_name TEXT NOT NULL, - pushkey TEXT NOT NULL, - ts BIGINT NOT NULL, - lang TEXT, - data TEXT, - last_stream_ordering INTEGER, - last_success BIGINT, - failing_since BIGINT, - UNIQUE(app_id, pushkey, user_name) -); -CREATE INDEX device_lists_outbound_pokes_stream ON device_lists_outbound_pokes( - stream_id -); -CREATE TABLE ratelimit_override( - user_id TEXT NOT NULL, - messages_per_second BIGINT, - burst_count BIGINT -); -CREATE UNIQUE INDEX ratelimit_override_idx ON ratelimit_override(user_id); -CREATE TABLE current_state_delta_stream( - stream_id BIGINT NOT NULL, - room_id TEXT NOT NULL, - type TEXT NOT NULL, - state_key TEXT NOT NULL, - event_id TEXT, - prev_event_id TEXT -); -CREATE INDEX current_state_delta_stream_idx ON current_state_delta_stream( - stream_id -); -CREATE TABLE device_lists_outbound_last_success( - destination TEXT NOT NULL, - user_id TEXT NOT NULL, - stream_id BIGINT NOT NULL -); -CREATE INDEX device_lists_outbound_last_success_idx ON device_lists_outbound_last_success( - destination, - user_id, - stream_id -); -CREATE TABLE user_directory_stream_pos( - Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE, - stream_id BIGINT, - CHECK(Lock='X') -); -CREATE TABLE IF NOT EXISTS "users_in_public_rooms"( - user_id TEXT NOT NULL, - room_id TEXT NOT NULL -); -CREATE VIRTUAL TABLE user_directory_search USING fts4( - user_id, - value -) -/* user_directory_search( - user_id, - value -) */; -CREATE TABLE IF NOT EXISTS 'user_directory_search_content'( - docid INTEGER PRIMARY KEY, - 'c0user_id', - 'c1value' -); -CREATE TABLE IF NOT EXISTS 'user_directory_search_segments'( - blockid INTEGER PRIMARY KEY, - block BLOB -); -CREATE TABLE IF NOT EXISTS 'user_directory_search_segdir'( - level INTEGER, - idx INTEGER, - start_block INTEGER, - leaves_end_block INTEGER, - end_block INTEGER, - root BLOB, - PRIMARY KEY(level, idx) -); -CREATE TABLE IF NOT EXISTS 'user_directory_search_docsize'( - docid INTEGER PRIMARY KEY, - size BLOB -); -CREATE TABLE IF NOT EXISTS 'user_directory_search_stat'(id INTEGER PRIMARY KEY, value BLOB); -CREATE TABLE blocked_rooms(room_id TEXT NOT NULL, user_id TEXT NOT NULL); -CREATE UNIQUE INDEX blocked_rooms_idx ON blocked_rooms(room_id); -CREATE TABLE users_who_share_rooms( - user_id TEXT NOT NULL, - other_user_id TEXT NOT NULL, - room_id TEXT NOT NULL, - share_private BOOLEAN NOT NULL -); -CREATE UNIQUE INDEX users_who_share_rooms_u_idx ON users_who_share_rooms( - user_id, - other_user_id -); -CREATE INDEX users_who_share_rooms_r_idx ON users_who_share_rooms(room_id); -CREATE INDEX users_who_share_rooms_o_idx ON users_who_share_rooms( - other_user_id -); -CREATE TABLE IF NOT EXISTS "local_media_repository_url_cache"( - url TEXT, - response_code INTEGER, - etag TEXT, - expires_ts BIGINT, - og TEXT, - media_id TEXT, - download_ts BIGINT -); -CREATE INDEX local_media_repository_url_cache_expires_idx ON local_media_repository_url_cache( - expires_ts -); -CREATE INDEX local_media_repository_url_cache_by_url_download_ts ON local_media_repository_url_cache( - url, - download_ts -); -CREATE INDEX local_media_repository_url_cache_media_idx ON local_media_repository_url_cache( - media_id -); -CREATE TABLE group_users( - group_id TEXT NOT NULL, - user_id TEXT NOT NULL, - is_admin BOOLEAN NOT NULL, - is_public BOOLEAN NOT NULL -); -CREATE TABLE group_invites(group_id TEXT NOT NULL, user_id TEXT NOT NULL); -CREATE TABLE group_rooms( - group_id TEXT NOT NULL, - room_id TEXT NOT NULL, - is_public BOOLEAN NOT NULL -); -CREATE TABLE group_summary_rooms( - group_id TEXT NOT NULL, - room_id TEXT NOT NULL, - category_id TEXT NOT NULL, - room_order BIGINT NOT NULL, - is_public BOOLEAN NOT NULL, - UNIQUE(group_id, category_id, room_id, room_order), - CHECK(room_order > 0) -); -CREATE UNIQUE INDEX group_summary_rooms_g_idx ON group_summary_rooms( - group_id, - room_id, - category_id -); -CREATE TABLE group_summary_room_categories( - group_id TEXT NOT NULL, - category_id TEXT NOT NULL, - cat_order BIGINT NOT NULL, - UNIQUE(group_id, category_id, cat_order), - CHECK(cat_order > 0) -); -CREATE TABLE group_room_categories( - group_id TEXT NOT NULL, - category_id TEXT NOT NULL, - profile TEXT NOT NULL, - is_public BOOLEAN NOT NULL, - UNIQUE(group_id, category_id) -); -CREATE TABLE group_summary_users( - group_id TEXT NOT NULL, - user_id TEXT NOT NULL, - role_id TEXT NOT NULL, - user_order BIGINT NOT NULL, - is_public BOOLEAN NOT NULL -); -CREATE INDEX group_summary_users_g_idx ON group_summary_users(group_id); -CREATE TABLE group_summary_roles( - group_id TEXT NOT NULL, - role_id TEXT NOT NULL, - role_order BIGINT NOT NULL, - UNIQUE(group_id, role_id, role_order), - CHECK(role_order > 0) -); -CREATE TABLE group_roles( - group_id TEXT NOT NULL, - role_id TEXT NOT NULL, - profile TEXT NOT NULL, - is_public BOOLEAN NOT NULL, - UNIQUE(group_id, role_id) -); -CREATE TABLE group_attestations_renewals( - group_id TEXT NOT NULL, - user_id TEXT NOT NULL, - valid_until_ms BIGINT NOT NULL -); -CREATE INDEX group_attestations_renewals_g_idx ON group_attestations_renewals( - group_id, - user_id -); -CREATE INDEX group_attestations_renewals_u_idx ON group_attestations_renewals( - user_id -); -CREATE INDEX group_attestations_renewals_v_idx ON group_attestations_renewals( - valid_until_ms -); -CREATE TABLE group_attestations_remote( - group_id TEXT NOT NULL, - user_id TEXT NOT NULL, - valid_until_ms BIGINT NOT NULL, - attestation_json TEXT NOT NULL -); -CREATE INDEX group_attestations_remote_g_idx ON group_attestations_remote( - group_id, - user_id -); -CREATE INDEX group_attestations_remote_u_idx ON group_attestations_remote( - user_id -); -CREATE INDEX group_attestations_remote_v_idx ON group_attestations_remote( - valid_until_ms -); -CREATE TABLE local_group_membership( - group_id TEXT NOT NULL, - user_id TEXT NOT NULL, - is_admin BOOLEAN NOT NULL, - membership TEXT NOT NULL, - is_publicised BOOLEAN NOT NULL, - content TEXT NOT NULL -); -CREATE INDEX local_group_membership_u_idx ON local_group_membership( - user_id, - group_id -); -CREATE INDEX local_group_membership_g_idx ON local_group_membership(group_id); -CREATE TABLE local_group_updates( - stream_id BIGINT NOT NULL, - group_id TEXT NOT NULL, - user_id TEXT NOT NULL, - type TEXT NOT NULL, - content TEXT NOT NULL -); -CREATE TABLE remote_profile_cache( - user_id TEXT NOT NULL, - displayname TEXT, - avatar_url TEXT, - last_check BIGINT NOT NULL -); -CREATE UNIQUE INDEX remote_profile_cache_user_id ON remote_profile_cache( - user_id -); -CREATE INDEX remote_profile_cache_time ON remote_profile_cache(last_check); -CREATE TABLE IF NOT EXISTS "deleted_pushers"( - stream_id BIGINT NOT NULL, - app_id TEXT NOT NULL, - pushkey TEXT NOT NULL, - user_id TEXT NOT NULL -); -CREATE INDEX deleted_pushers_stream_id ON deleted_pushers(stream_id); -CREATE TABLE IF NOT EXISTS "groups"( - group_id TEXT NOT NULL, - name TEXT, - avatar_url TEXT, - short_description TEXT, - long_description TEXT, - is_public BOOL NOT NULL , - join_policy TEXT NOT NULL DEFAULT 'invite' -); -CREATE UNIQUE INDEX groups_idx ON groups(group_id); -CREATE TABLE IF NOT EXISTS "user_directory"( - user_id TEXT NOT NULL, - room_id TEXT, - display_name TEXT, - avatar_url TEXT -); -CREATE INDEX user_directory_room_idx ON user_directory(room_id); -CREATE UNIQUE INDEX user_directory_user_idx ON user_directory(user_id); -CREATE INDEX users_in_public_rooms_room_idx ON users_in_public_rooms(room_id); -CREATE UNIQUE INDEX users_in_public_rooms_user_idx ON users_in_public_rooms( - user_id -); -CREATE TABLE event_push_actions_staging( - event_id TEXT NOT NULL, - user_id TEXT NOT NULL, - actions TEXT NOT NULL, - notif SMALLINT NOT NULL, - highlight SMALLINT NOT NULL -); -CREATE INDEX event_push_actions_staging_id ON event_push_actions_staging( - event_id -); -CREATE TABLE users_pending_deactivation(user_id TEXT NOT NULL); -CREATE UNIQUE INDEX group_invites_g_idx ON group_invites(group_id, user_id); -CREATE UNIQUE INDEX group_users_g_idx ON group_users(group_id, user_id); -CREATE INDEX group_users_u_idx ON group_users(user_id); -CREATE INDEX group_invites_u_idx ON group_invites(user_id); -CREATE UNIQUE INDEX group_rooms_g_idx ON group_rooms(group_id, room_id); -CREATE INDEX group_rooms_r_idx ON group_rooms(room_id); -CREATE TABLE user_daily_visits( - user_id TEXT NOT NULL, - device_id TEXT, - timestamp BIGINT NOT NULL -); -CREATE INDEX user_daily_visits_uts_idx ON user_daily_visits( - user_id, - timestamp -); -CREATE INDEX user_daily_visits_ts_idx ON user_daily_visits(timestamp); -CREATE TABLE erased_users(user_id TEXT NOT NULL); -CREATE UNIQUE INDEX erased_users_user ON erased_users(user_id); -CREATE TABLE monthly_active_users( - user_id TEXT NOT NULL, - timestamp BIGINT NOT NULL -); -CREATE UNIQUE INDEX monthly_active_users_users ON monthly_active_users( - user_id -); -CREATE INDEX monthly_active_users_time_stamp ON monthly_active_users( - timestamp -); -CREATE TABLE IF NOT EXISTS "e2e_room_keys_versions"( - user_id TEXT NOT NULL, - version BIGINT NOT NULL, - algorithm TEXT NOT NULL, - auth_data TEXT NOT NULL, - deleted SMALLINT DEFAULT 0 NOT NULL -); -CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions( - user_id, - version -); -CREATE TABLE IF NOT EXISTS "e2e_room_keys"( - user_id TEXT NOT NULL, - room_id TEXT NOT NULL, - session_id TEXT NOT NULL, - version BIGINT NOT NULL, - first_message_index INT, - forwarded_count INT, - is_verified BOOLEAN, - session_data TEXT NOT NULL -); -CREATE UNIQUE INDEX e2e_room_keys_idx ON e2e_room_keys( - user_id, - room_id, - session_id -); -CREATE INDEX access_tokens_device_id ON access_tokens(user_id, device_id); -CREATE INDEX user_ips_device_id ON user_ips(user_id, device_id, last_seen); -CREATE INDEX event_contains_url_index ON events( - room_id, - topological_ordering, - stream_ordering -); -CREATE INDEX event_push_actions_u_highlight ON event_push_actions( - user_id, - stream_ordering -); -CREATE INDEX event_push_actions_highlights_index ON event_push_actions( - user_id, - room_id, - topological_ordering, - stream_ordering -); -CREATE INDEX current_state_events_member_index ON current_state_events( - state_key -); -CREATE INDEX device_inbox_stream_id_user_id ON device_inbox( - stream_id, - user_id -); -CREATE INDEX device_lists_stream_user_id ON device_lists_stream( - user_id, - device_id -); -CREATE INDEX local_media_repository_url_idx ON local_media_repository( - created_ts -); -CREATE INDEX user_ips_last_seen ON user_ips(user_id, last_seen); -CREATE INDEX user_ips_last_seen_only ON user_ips(last_seen); -CREATE INDEX users_creation_ts ON users(creation_ts); -CREATE INDEX event_to_state_groups_sg_index ON event_to_state_groups( - state_group -); -CREATE UNIQUE INDEX device_lists_remote_cache_unique_id ON device_lists_remote_cache( - user_id, - device_id -); -CREATE UNIQUE INDEX user_ips_device_unique_id ON user_ips( - user_id, - access_token, - ip, - user_agent, - device_id -); -CREATE INDEX state_groups_state_type_idx ON state_groups_state( - state_group, - type, - state_key -); -CREATE UNIQUE INDEX device_lists_remote_extremeties_unique_idx ON device_lists_remote_extremeties( - user_id -); diff --git a/synapse/storage/schema/full_schemas/54/zzz-inserts.sql b/synapse/storage/schema/full_schemas/54/zzz-inserts.sql deleted file mode 100644 index 859691b0fb75..000000000000 --- a/synapse/storage/schema/full_schemas/54/zzz-inserts.sql +++ /dev/null @@ -1,12 +0,0 @@ -INSERT INTO user_directory_stream_pos (stream_id) VALUES (null); - -INSERT INTO appservice_stream_position (stream_ordering) - SELECT COALESCE(MAX(stream_ordering), 0) FROM events; - -INSERT INTO device_max_stream_id (stream_id) - SELECT COALESCE(MAX(stream_id), 0) FROM device_inbox; - -INSERT INTO event_push_summary_stream_ordering (stream_ordering) VALUES (0); - -INSERT INTO federation_stream_position (type, stream_id) VALUES ('federation', -1); -INSERT INTO federation_stream_position (type, stream_id) SELECT 'events', coalesce(max(stream_ordering), -1) FROM events; From 238b8d4d5e52820afcc924f5435ad9636d548765 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 18 Jan 2019 17:20:09 +1100 Subject: [PATCH 34/40] things we don't need --- synapse/storage/_base.py | 18 +----------------- synapse/storage/monthly_active_users.py | 5 +---- synapse/storage/prepare_database.py | 22 +++------------------- 3 files changed, 5 insertions(+), 40 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 5c5984e35587..4d2d9eed30b5 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -690,23 +690,7 @@ def _simple_upsert_txn_native_upsert( ", ".join(k for k in keyvalues), ", ".join(k + "=EXCLUDED." + k for k in values), ) - try: - txn.execute(sql, list(allvalues.values())) - except self.database_engine.module.OperationalError as e: - # We only care about serialization errors, so check for it - if e.args[0] == "could not serialize access due to concurrent update": - # A concurrent update problem is when we try and do a native - # UPSERT but the row has changed from under us. We can either - # retry, or give up if asked to do so. - if best_effort: - # If it's a concurrent-update problem, and this is marked as - # 'best effort' (i.e. if there's a race, then the one we - # raced with will suffice), then pretend that we succeeded. - return False - else: - # Otherwise, raise, because it's a real OperationalError and we - # will need to be rolled back and retried. - raise + txn.execute(sql, list(allvalues.values())) # One-tuple, which is a boolean for insertion or not res = txn.fetchone() diff --git a/synapse/storage/monthly_active_users.py b/synapse/storage/monthly_active_users.py index cff3ad0f3094..d6fc8edd4c36 100644 --- a/synapse/storage/monthly_active_users.py +++ b/synapse/storage/monthly_active_users.py @@ -237,9 +237,7 @@ def upsert_monthly_active_user_txn(self, txn, user_id): # Am consciously deciding to lock the table on the basis that is ought # never be a big table and alternative approaches (batching multiple # upserts into a single txn) introduced a lot of extra complexity. - # See https://github.com/matrix-org/synapse/issues/3854 for more. - # If we support native upserts, we'll not lock, but also not retry - # on any races, by setting best_effort=True. + # See https://github.com/matrix-org/synapse/issues/3854 for more is_insert = self._simple_upsert_txn( txn, table="monthly_active_users", @@ -249,7 +247,6 @@ def upsert_monthly_active_user_txn(self, txn, user_id): values={ "timestamp": int(self._clock.time_msec()), }, - best_effort=True, ) return is_insert diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 48dd64e9a0ec..fa36daac524d 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -20,14 +20,12 @@ import os import re -from synapse.storage.engines.postgres import PostgresEngine - logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 54 +SCHEMA_VERSION = 53 dir_path = os.path.abspath(os.path.dirname(__file__)) @@ -117,24 +115,13 @@ def _setup_new_database(cur, database_engine): valid_dirs = [] pattern = re.compile(r"^\d+(\.sql)?$") - - if isinstance(database_engine, PostgresEngine): - specific = "postgres" - else: - specific = "sqlite" - - specific_pattern = re.compile(r"^\d+(\.sql." + specific + r")?$") - for filename in directory_entries: - match = pattern.match(filename) or specific_pattern.match(filename) + match = pattern.match(filename) abs_path = os.path.join(current_dir, filename) if match and os.path.isdir(abs_path): ver = int(match.group(0)) if ver <= SCHEMA_VERSION: valid_dirs.append((ver, abs_path)) - elif filename == "README.md": - # Ignore the readme - pass else: logger.warn("Unexpected entry in 'full_schemas': %s", filename) @@ -149,10 +136,7 @@ def _setup_new_database(cur, database_engine): directory_entries = os.listdir(sql_dir) - for filename in sorted( - fnmatch.filter(directory_entries, "*.sql") - + fnmatch.filter(directory_entries, "*.sql." + specific) - ): + for filename in fnmatch.filter(directory_entries, "*.sql"): sql_loc = os.path.join(sql_dir, filename) logger.debug("Applying schema %s", sql_loc) executescript(cur, sql_loc) From 078535ae0873ffd8c3888880e5df569f79adb8bc Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 18 Jan 2019 23:53:04 +1100 Subject: [PATCH 35/40] things we don't need --- synapse/storage/schema/full_schemas/README.md | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 synapse/storage/schema/full_schemas/README.md diff --git a/synapse/storage/schema/full_schemas/README.md b/synapse/storage/schema/full_schemas/README.md deleted file mode 100644 index a4ad0c149492..000000000000 --- a/synapse/storage/schema/full_schemas/README.md +++ /dev/null @@ -1,16 +0,0 @@ -Building full schema dumps -========================== - -Postgres --------- - -$ pg_dump --schema-only --no-comments --no-tablespaces --no-acl --no-owner ${DATABASE_NAME} > full.sql.postgres - -In the resulting file, delete all SET commands from the top of the file. - -SQLite ------- - -$ sqlite3 $DATABASE_FILE ".schema" > full.sql.sqlite - -Delete the CREATE statements for "schema_version", "applied_schema_deltas", and "applied_module_schemas". \ No newline at end of file From 9392c556779d09dfedd4049fbe631a1ceb69233e Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 18 Jan 2019 23:54:46 +1100 Subject: [PATCH 36/40] revert --- tests/unittest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittest.py b/tests/unittest.py index 7273c73ce038..cda549c78329 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -96,7 +96,7 @@ def __init__(self, methodName, *args, **kwargs): method = getattr(self, methodName) - level = getattr(method, "loglevel", getattr(self, "loglevel", logging.DEBUG)) + level = getattr(method, "loglevel", getattr(self, "loglevel", logging.WARNING)) @around(self) def setUp(orig): From ee92e1ab1a8824405b04f9411a63e61306e0bab1 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 23 Jan 2019 22:47:43 +1100 Subject: [PATCH 37/40] remove uses where we care about what an upsert returns --- synapse/storage/_base.py | 7 +---- synapse/storage/engines/sqlite.py | 10 ++----- synapse/storage/pusher.py | 9 ++++-- synapse/storage/user_directory.py | 48 +++++++++++++++++++++---------- 4 files changed, 43 insertions(+), 31 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 4d2d9eed30b5..48420a69c82f 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -681,8 +681,7 @@ def _simple_upsert_txn_native_upsert( sql = ( "INSERT INTO %s (%s) VALUES (%s) " - "ON CONFLICT (%s) DO UPDATE SET %s " - "RETURNING (xmax = 0) AS inserted" + "ON CONFLICT (%s) DO UPDATE SET %s" ) % ( table, ", ".join(k for k in allvalues), @@ -692,10 +691,6 @@ def _simple_upsert_txn_native_upsert( ) txn.execute(sql, list(allvalues.values())) - # One-tuple, which is a boolean for insertion or not - res = txn.fetchone() - return res[0] - def _simple_select_one(self, table, keyvalues, retcols, allow_none=False, desc="_simple_select_one"): """Executes a SELECT query on the named table, which is expected to diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py index 50b80da3261f..c64d73ff21c9 100644 --- a/synapse/storage/engines/sqlite.py +++ b/synapse/storage/engines/sqlite.py @@ -15,6 +15,7 @@ import struct import threading +from sqlite3 import sqlite_version_info from synapse.storage.prepare_database import prepare_database @@ -36,14 +37,7 @@ def can_native_upsert(self): Do we support native UPSERTs? This requires SQLite3 3.24+, plus some more work we haven't done yet to tell what was inserted vs updated. """ - # SQLite currently doesn't have a way of telling if an UPSERT ended up - # with an INSERT or an UPDATE - # To enable, uncomment: - # - # from sqlite3 import sqlite_version_info - # return sqlite_version_info >= (3, 24, 0) - - return False + return sqlite_version_info >= (3, 24, 0) def check_database(self, txn): pass diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py index 2743b52bad66..134297e28464 100644 --- a/synapse/storage/pusher.py +++ b/synapse/storage/pusher.py @@ -215,7 +215,7 @@ def add_pusher(self, user_id, access_token, kind, app_id, with self._pushers_id_gen.get_next() as stream_id: # no need to lock because `pushers` has a unique key on # (app_id, pushkey, user_name) so _simple_upsert will retry - newly_inserted = yield self._simple_upsert( + yield self._simple_upsert( table="pushers", keyvalues={ "app_id": app_id, @@ -238,7 +238,12 @@ def add_pusher(self, user_id, access_token, kind, app_id, lock=False, ) - if newly_inserted: + user_has_pusher = self.get_if_user_has_pusher.cache.get( + (user_id,), None, update_metrics=False + ) + + if user_has_pusher is not True: + # invalidate, since we the user might not have had a pusher before yield self.runInteraction( "add_pusher", self._invalidate_cache_and_stream, diff --git a/synapse/storage/user_directory.py b/synapse/storage/user_directory.py index a8781b0e5d5b..61b03fbbdb3f 100644 --- a/synapse/storage/user_directory.py +++ b/synapse/storage/user_directory.py @@ -168,14 +168,14 @@ def _update_profile_in_user_dir_txn(txn): if isinstance(self.database_engine, PostgresEngine): # We weight the localpart most highly, then display name and finally # server name - if new_entry: + if self.database_engine.can_native_upsert: sql = """ INSERT INTO user_directory_search(user_id, vector) VALUES (?, setweight(to_tsvector('english', ?), 'A') || setweight(to_tsvector('english', ?), 'D') || setweight(to_tsvector('english', COALESCE(?, '')), 'B') - ) + ) ON CONFLICT (user_id) DO UPDATE SET vector=EXCLUDED.vector """ txn.execute( sql, @@ -185,20 +185,38 @@ def _update_profile_in_user_dir_txn(txn): ) ) else: - sql = """ - UPDATE user_directory_search - SET vector = setweight(to_tsvector('english', ?), 'A') - || setweight(to_tsvector('english', ?), 'D') - || setweight(to_tsvector('english', COALESCE(?, '')), 'B') - WHERE user_id = ? - """ - txn.execute( - sql, - ( - get_localpart_from_id(user_id), get_domain_from_id(user_id), - display_name, user_id, + if new_entry: + sql = """ + INSERT INTO user_directory_search(user_id, vector) + VALUES (?, + setweight(to_tsvector('english', ?), 'A') + || setweight(to_tsvector('english', ?), 'D') + || setweight(to_tsvector('english', COALESCE(?, '')), 'B') + ) + """ + txn.execute( + sql, + ( + user_id, get_localpart_from_id(user_id), + get_domain_from_id(user_id), display_name, + ) + ) + else: + sql = """ + UPDATE user_directory_search + SET vector = setweight(to_tsvector('english', ?), 'A') + || setweight(to_tsvector('english', ?), 'D') + || setweight(to_tsvector('english', COALESCE(?, '')), 'B') + WHERE user_id = ? + """ + txn.execute( + sql, + ( + get_localpart_from_id(user_id), + get_domain_from_id(user_id), + display_name, user_id, + ) ) - ) elif isinstance(self.database_engine, Sqlite3Engine): value = "%s %s" % (user_id, display_name,) if display_name else user_id self._simple_upsert_txn( From b7c35114c8573498339d8a5979d7ebcdf7eeeead Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 23 Jan 2019 22:59:32 +1100 Subject: [PATCH 38/40] Update MANIFEST.in --- MANIFEST.in | 2 -- 1 file changed, 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 70e40040e59e..7e4c031b7988 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,9 +9,7 @@ include demo/*.py include demo/*.sh recursive-include synapse/storage/schema *.sql -recursive-include synapse/storage/schema *.sql.* recursive-include synapse/storage/schema *.py -recursive-include synapse/storage/schema *.md recursive-include docs * recursive-include scripts * From d53a60bcb167387cefc4d3f4bbf4640e47b566b4 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 23 Jan 2019 23:24:51 +1100 Subject: [PATCH 39/40] Update 4306.misc --- changelog.d/4306.misc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/4306.misc b/changelog.d/4306.misc index 7f48b02fbf96..58130b61908b 100644 --- a/changelog.d/4306.misc +++ b/changelog.d/4306.misc @@ -1 +1 @@ -Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+. +Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+ and SQLite 3.24+. From 1b1d7826bde8bad877e619bb465b0792b2c211f6 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 24 Jan 2019 20:36:20 +1100 Subject: [PATCH 40/40] update due to review --- synapse/storage/_base.py | 36 +++++++++++++++++++++++-------- synapse/storage/user_directory.py | 11 ++++++++-- 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 48420a69c82f..254fdc04c650 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -536,8 +536,7 @@ def _simple_upsert( values, insertion_values={}, desc="_simple_upsert", - lock=True, - best_effort=False, + lock=True ): """ @@ -558,10 +557,10 @@ def _simple_upsert( insertion_values (dict): additional key/values to use only when inserting lock (bool): True to lock the table when doing the upsert. - best_effort (bool): If we run into a transaction error, do we stop trying? Returns: - Deferred(bool): True if a new entry was created, False if an - existing one was updated. + Deferred(None or bool): Native upserts always return None. Emulated + upserts return True if a new entry was created, False if an existing + one was updated. """ attempts = 0 while True: @@ -574,7 +573,6 @@ def _simple_upsert( values, insertion_values, lock=lock, - best_effort=best_effort, ) defer.returnValue(result) except self.database_engine.module.IntegrityError as e: @@ -597,11 +595,23 @@ def _simple_upsert_txn( values, insertion_values={}, lock=True, - best_effort=False, ): """ Pick the UPSERT method which works best on the platform. Either the native one (Pg9.5+, recent SQLites), or fall back to an emulated method. + + Args: + txn: The transaction to use. + table (str): The table to upsert into + keyvalues (dict): The unique key tables and their new values + values (dict): The nonunique columns and their new values + insertion_values (dict): additional key/values to use only when + inserting + lock (bool): True to lock the table when doing the upsert. + Returns: + Deferred(None or bool): Native upserts always return None. Emulated + upserts return True if a new entry was created, False if an existing + one was updated. """ if ( self.database_engine.can_native_upsert @@ -613,7 +623,6 @@ def _simple_upsert_txn( keyvalues, values, insertion_values=insertion_values, - best_effort=best_effort, ) else: return self._simple_upsert_txn_emulated( @@ -669,10 +678,19 @@ def _getwhere(key): return True def _simple_upsert_txn_native_upsert( - self, txn, table, keyvalues, values, insertion_values={}, best_effort=False + self, txn, table, keyvalues, values, insertion_values={} ): """ Use the native UPSERT functionality in recent PostgreSQL versions. + + Args: + table (str): The table to upsert into + keyvalues (dict): The unique key tables and their new values + values (dict): The nonunique columns and their new values + insertion_values (dict): additional key/values to use only when + inserting + Returns: + None """ allvalues = {} allvalues.update(keyvalues) diff --git a/synapse/storage/user_directory.py b/synapse/storage/user_directory.py index 61b03fbbdb3f..ce4821226599 100644 --- a/synapse/storage/user_directory.py +++ b/synapse/storage/user_directory.py @@ -185,7 +185,10 @@ def _update_profile_in_user_dir_txn(txn): ) ) else: - if new_entry: + # TODO: Remove this code after we've bumped the minimum version + # of postgres to always support upserts, so we can get rid of + # `new_entry` usage + if new_entry is True: sql = """ INSERT INTO user_directory_search(user_id, vector) VALUES (?, @@ -201,7 +204,7 @@ def _update_profile_in_user_dir_txn(txn): get_domain_from_id(user_id), display_name, ) ) - else: + elif new_entry is False: sql = """ UPDATE user_directory_search SET vector = setweight(to_tsvector('english', ?), 'A') @@ -217,6 +220,10 @@ def _update_profile_in_user_dir_txn(txn): display_name, user_id, ) ) + else: + raise RuntimeError( + "upsert returned None when 'can_native_upsert' is False" + ) elif isinstance(self.database_engine, Sqlite3Engine): value = "%s %s" % (user_id, display_name,) if display_name else user_id self._simple_upsert_txn(