Add native Matrix edit support

Warning: may break everything and/or edit your cat

Dockerfile

@@ -10,7 +10,6 @@ RUN apk add --no-cache \
       py3-virtualenv \
       py3-pillow \
       py3-aiohttp \
-      py3-lxml \
       py3-magic \
       py3-sqlalchemy \
       py3-markdown \

ROADMAP.md

@@ -3,6 +3,7 @@
 * Matrix → Telegram
   * [x] Message content (text, formatting, files, etc..)
   * [x] Message redactions
+  * [x] Message edits
   * [ ] ‡ Message history
   * [x] Presence
   * [x] Typing notifications

alembic/versions/9e9c89b0b877_add_edit_index_to_messages.py

@@ -0,0 +1,50 @@
+"""Add edit index to messages
+
+Revision ID: 9e9c89b0b877
+Revises: 17574c57f3f8
+Create Date: 2019-05-29 15:28:23.128377
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '9e9c89b0b877'
+down_revision = '17574c57f3f8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('_message_temp',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.Column('edit_index', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space', 'edit_index'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"))
+    c = op.get_bind()
+    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space, edit_index) "
+              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space, 0 "
+              "FROM message")
+    c.execute("DROP TABLE message")
+    c.execute("ALTER TABLE _message_temp RENAME TO message")
+
+
+
+def downgrade():
+    op.create_table('_message_temp',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"))
+    c = op.get_bind()
+    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space) "
+              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space "
+              "FROM portal")
+    c.execute("DROP TABLE message")
+    c.execute("ALTER TABLE _message_temp RENAME TO message")

example-config.yaml

@@ -126,11 +126,6 @@ bridge:
     # Only enable this if your displayname_template has some static part that the bridge can use to
     # reliably identify what is a plaintext highlight.
     plaintext_highlights: false
-    # Show message editing as a reply to the original message.
-    # If this is false, message edits are not shown at all, as Matrix does not support editing yet.
-    edits_as_replies: true
-    # Highlight changed/added parts in edits. Requires lxml.
-    highlight_edits: false
     # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
     public_portals: true
     # Whether or not to fetch and handle Telegram updates at startup from the time the bridge was down.

mautrix_telegram/abstract_user.py

@@ -252,7 +252,7 @@ async def update_read_receipt(self, update: UpdateReadHistoryOutbox) -> None:
             return
 
         # We check that these are user read receipts, so tg_space is always the user ID.
-        message = DBMessage.get_by_tgid(TelegramID(update.max_id), self.tgid)
+        message = DBMessage.get_one_by_tgid(TelegramID(update.max_id), self.tgid, edit_index=-1)
         if not message:
             return
 
@@ -336,7 +336,8 @@ def get_message_details(self, update: UpdateMessage) -> Tuple[UpdateMessageConte
         return update, sender, portal
 
     @staticmethod
-    async def _try_redact(portal: po.Portal, message: DBMessage) -> None:
+    async def _try_redact(message: DBMessage) -> None:
+        portal = po.Portal.get_by_mxid(message.mx_room)
         if not portal:
             return
         try:
@@ -348,30 +349,26 @@ async def delete_message(self, update: UpdateDeleteMessages) -> None:
         if len(update.messages) > MAX_DELETIONS:
             return
 
-        for message in update.messages:
-            message = DBMessage.get_by_tgid(TelegramID(message), self.tgid)
-            if not message:
-                continue
-            message.delete()
-            number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
-            if number_left == 0:
-                portal = po.Portal.get_by_mxid(message.mx_room)
-                await self._try_redact(portal, message)
+        for message_id in update.messages:
+            messages = DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid)
+            for message in messages:
+                message.delete()
+                number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
+                if number_left == 0:
+                    portal = po.Portal.get_by_mxid(message.mx_room)
+                    await self._try_redact(message)
 
     async def delete_channel_message(self, update: UpdateDeleteChannelMessages) -> None:
         if len(update.messages) > MAX_DELETIONS:
             return
 
-        portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
-        if not portal:
-            return
+        channel_id = TelegramID(update.channel_id)
 
-        for message in update.messages:
-            message = DBMessage.get_by_tgid(TelegramID(message), portal.tgid)
-            if not message:
-                continue
-            message.delete()
-            await self._try_redact(portal, message)
+        for message_id in update.messages:
+            messages = DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id)
+            for message in messages:
+                message.delete()
+                await self._try_redact(message)
 
     async def update_message(self, original_update: UpdateMessage) -> None:
         update, sender, portal = self.get_message_details(original_update)
@@ -397,10 +394,7 @@ async def update_message(self, original_update: UpdateMessage) -> None:
 
         user = sender.tgid if sender else "admin"
         if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
-            if config["bridge.edits_as_replies"]:
-                self.log.debug("Handling edit %s to %s by %s", update, portal.tgid_log, user)
-                return await portal.handle_telegram_edit(self, sender, update)
-            return
+            return await portal.handle_telegram_edit(self, sender, update)
 
         self.log.debug("Handling message %s to %s by %s", update, portal.tgid_log, user)
         return await portal.handle_telegram_message(self, sender, update)

mautrix_telegram/commands/portal/config.py

@@ -77,7 +77,6 @@ def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[Dict]:
 def config_defaults(evt: CommandEvent) -> Awaitable[Dict]:
     stream = StringIO()
     yaml.dump({
-        "edits_as_replies": evt.config["bridge.edits_as_replies"],
         "bridge_notices": {
             "default": evt.config["bridge.bridge_notices.default"],
             "exceptions": evt.config["bridge.bridge_notices.exceptions"],

mautrix_telegram/commands/telegram/auth.py

@@ -33,8 +33,8 @@
                  help_text="Check if you're logged into Telegram.")
 async def ping(evt: CommandEvent) -> Optional[Dict]:
     me = await evt.sender.client.get_me() if await evt.sender.is_logged_in() else None
-    human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
     if me:
+        human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
         return await evt.reply(f"You're logged in as {human_tg_id}")
     else:
         return await evt.reply("You're not logged in.")

mautrix_telegram/commands/telegram/misc.py

@@ -191,7 +191,7 @@ async def _parse_encoded_msgid(user: AbstractUser, enc_id: str, type_name: str
         raise MessageIDError(f"Invalid {type_name} ID (format)") from e
 
     if peer_type == PEER_TYPE_CHAT:
-        orig_msg = DBMessage.get_by_tgid(msg_id, space)
+        orig_msg = DBMessage.get_one_by_tgid(msg_id, space)
         if not orig_msg:
             raise MessageIDError(f"Invalid {type_name} ID (original message not found in db)")
         new_msg = DBMessage.get_by_mxid(orig_msg.mxid, orig_msg.mx_room, user.tgid)

mautrix_telegram/config.py

@@ -199,8 +199,6 @@ def copy_dict(from_path, to_path=None, override_existing_map=True) -> None:
         copy("bridge.sync_matrix_state")
         copy("bridge.allow_matrix_login")
         copy("bridge.plaintext_highlights")
-        copy("bridge.edits_as_replies")
-        copy("bridge.highlight_edits")
         copy("bridge.public_portals")
         copy("bridge.catch_up")
         copy("bridge.sync_with_custom_puppets")

mautrix_telegram/db/message.py

@@ -14,7 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, select
+from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, desc, select
 from sqlalchemy.engine.result import RowProxy
 from typing import Optional, List
 
@@ -29,25 +29,44 @@ class Message(Base):
     mx_room = Column(String)  # type: MatrixRoomID
     tgid = Column(Integer, primary_key=True)  # type: TelegramID
     tg_space = Column(Integer, primary_key=True)  # type: TelegramID
+    edit_index = Column(Integer, primary_key=True)  # type: int
 
     __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"),)
 
     @classmethod
     def _one_or_none(cls, rows: RowProxy) -> Optional['Message']:
         try:
-            mxid, mx_room, tgid, tg_space = next(rows)
-            return cls(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space)
+            mxid, mx_room, tgid, tg_space, edit_index = next(rows)
+            return cls(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space,
+                       edit_index=edit_index)
         except StopIteration:
             return None
 
     @staticmethod
     def _all(rows: RowProxy) -> List['Message']:
-        return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3])
+        return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3],
+                        edit_index=row[4])
                 for row in rows]
 
     @classmethod
-    def get_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> Optional['Message']:
-        return cls._select_one_or_none(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
+    def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> List['Message']:
+        return cls._all(cls.db.execute(cls.t.select().where(and_(cls.c.tgid == tgid,
+                                                                 cls.c.tg_space == tg_space))))
+
+    @classmethod
+    def get_one_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID, edit_index: int = 0
+                        ) -> Optional['Message']:
+        query = cls.t.select()
+        if edit_index < 0:
+            query = (query
+                     .where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
+                     .order_by(desc(cls.c.edit_index))
+                     .limit(1)
+                     .offset(-edit_index - 1))
+        else:
+            query = query.where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space,
+                                     cls.c.edit_index == edit_index))
+        return cls._one_or_none(cls.db.execute(query))
 
     @classmethod
     def count_spaces_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID) -> int:
@@ -67,10 +86,12 @@ def get_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID, tg_space: Teleg
                                             cls.c.tg_space == tg_space))
 
     @classmethod
-    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, **values) -> None:
+    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, s_edit_index: int,
+                       **values) -> None:
         with cls.db.begin() as conn:
             conn.execute(cls.t.update()
-                         .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space))
+                         .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space,
+                                     cls.c.edit_index == s_edit_index))
                          .values(**values))
 
     @classmethod
@@ -82,9 +103,11 @@ def update_by_mxid(cls, s_mxid: MatrixEventID, s_mx_room: MatrixRoomID, **values
 
     @property
     def _edit_identity(self):
-        return and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space)
+        return and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space,
+                    self.c.edit_index == self.edit_index)
 
     def insert(self) -> None:
         with self.db.begin() as conn:
             conn.execute(self.t.insert().values(mxid=self.mxid, mx_room=self.mx_room,
-                                                tgid=self.tgid, tg_space=self.tg_space))
+                                                tgid=self.tgid, tg_space=self.tg_space,
+                                                edit_index=self.edit_index))

mautrix_telegram/formatter/__init__.py

@@ -1,9 +1,8 @@
 from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
                           init_mx)
-from .from_telegram import (telegram_reply_to_matrix, telegram_to_matrix, init_tg)
+from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix
 from .. import context as c
 
 
 def init(context: c.Context) -> None:
     init_mx(context)
-    init_tg(context)

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -87,25 +87,28 @@ def matrix_to_telegram(html: str) -> ParsedMessage:
 
 def matrix_reply_to_telegram(content: Dict[str, Any], tg_space: TelegramID,
                              room_id: Optional[MatrixRoomID] = None) -> Optional[TelegramID]:
+    relates_to = content.get("m.relates_to", None) or {}
+    if not relates_to:
+        return None
+    reply = (relates_to if relates_to.get("rel_type", None) == "m.reference"
+             else relates_to.get("m.in_reply_to", None) or {})
+    if not reply:
+        return None
+    room_id = room_id or reply.get("room_id", None)
+    event_id = reply.get("event_id", None)
+    if not event_id:
+        return
+
     try:
-        reply = (content.get("m.relates_to", None) or {}).get("m.in_reply_to", {})
-        if not reply:
-            return None
-        room_id = room_id or reply["room_id"]
-        event_id = reply["event_id"]
-
-        try:
-            if content["format"] == "org.matrix.custom.html":
-                content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
-        except KeyError:
-            pass
-        content["body"] = trim_reply_fallback_text(content["body"])
-
-        message = DBMessage.get_by_mxid(event_id, room_id, tg_space)
-        if message:
-            return message.tgid
+        if content["format"] == "org.matrix.custom.html":
+            content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
     except KeyError:
         pass
+    content["body"] = trim_reply_fallback_text(content["body"])
+
+    message = DBMessage.get_by_mxid(event_id, room_id, tg_space)
+    if message:
+        return message.tgid
     return None