From 2e6567c7801537e1f7aa03cee066883ea827063f Mon Sep 17 00:00:00 2001
From: Patrick Cloke <patrickc@matrix.org>
Date: Wed, 15 Dec 2021 13:45:39 -0500
Subject: [PATCH 1/3] Do not bundle aggregations for APIs which shouldn't
 include them.

---
 synapse/handlers/search.py           | 10 ++++++----
 synapse/rest/client/events.py        |  4 +++-
 synapse/rest/client/notifications.py |  1 +
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py
index ab7eaab2fb56..1624abc07866 100644
--- a/synapse/handlers/search.py
+++ b/synapse/handlers/search.py
@@ -421,10 +421,10 @@ async def search(
 
         for context in contexts.values():
             context["events_before"] = await self._event_serializer.serialize_events(
-                context["events_before"], time_now
+                context["events_before"], time_now, bundle_aggregations=False
             )
             context["events_after"] = await self._event_serializer.serialize_events(
-                context["events_after"], time_now
+                context["events_after"], time_now, bundle_aggregations=False
             )
 
         state_results = {}
@@ -442,7 +442,9 @@ async def search(
                 {
                     "rank": rank_map[e.event_id],
                     "result": (
-                        await self._event_serializer.serialize_event(e, time_now)
+                        await self._event_serializer.serialize_event(
+                            e, time_now, bundle_aggregations=False
+                        )
                     ),
                     "context": contexts.get(e.event_id, {}),
                 }
@@ -458,7 +460,7 @@ async def search(
             s = {}
             for room_id, state_events in state_results.items():
                 s[room_id] = await self._event_serializer.serialize_events(
-                    state_events, time_now
+                    state_events, time_now, bundle_aggregations=False
                 )
 
             rooms_cat_res["state"] = s
diff --git a/synapse/rest/client/events.py b/synapse/rest/client/events.py
index 13b72a045a4a..cd4ff616a1e9 100644
--- a/synapse/rest/client/events.py
+++ b/synapse/rest/client/events.py
@@ -91,7 +91,9 @@ async def on_GET(
 
         time_now = self.clock.time_msec()
         if event:
-            result = await self._event_serializer.serialize_event(event, time_now)
+            result = await self._event_serializer.serialize_event(
+                event, time_now, bundle_aggregations=False
+            )
             return 200, result
         else:
             return 404, "Event not found."
diff --git a/synapse/rest/client/notifications.py b/synapse/rest/client/notifications.py
index b12a332776e4..1de2dd6f24ab 100644
--- a/synapse/rest/client/notifications.py
+++ b/synapse/rest/client/notifications.py
@@ -75,6 +75,7 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
                     await self._event_serializer.serialize_event(
                         notif_events[pa["event_id"]],
                         self.clock.time_msec(),
+                        bundle_aggregations=False,
                         event_format=format_event_for_client_v2_without_room_id,
                     )
                 ),

From 8d9dda590ed04fa8baed44d90b6ee7d5b075671b Mon Sep 17 00:00:00 2001
From: Patrick Cloke <patrickc@matrix.org>
Date: Thu, 16 Dec 2021 10:07:52 -0500
Subject: [PATCH 2/3] Newsfragment

---
 changelog.d/11592.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 changelog.d/11592.bugfix

diff --git a/changelog.d/11592.bugfix b/changelog.d/11592.bugfix
new file mode 100644
index 000000000000..4116e5dd7c7b
--- /dev/null
+++ b/changelog.d/11592.bugfix
@@ -0,0 +1 @@
+Fix a long-standing bug where responses included bundled aggregations when they should not, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675).

From 6bce18de9ded1aa4163ceb3879105af22f6f17e6 Mon Sep 17 00:00:00 2001
From: Patrick Cloke <patrickc@matrix.org>
Date: Mon, 20 Dec 2021 13:04:46 -0500
Subject: [PATCH 3/3] Swap the default of bundle_aggregations to false.

---
 synapse/events/utils.py              |  2 +-
 synapse/handlers/events.py           |  2 --
 synapse/handlers/initial_sync.py     | 18 ++++--------------
 synapse/handlers/message.py          |  4 +++-
 synapse/handlers/pagination.py       |  5 ++++-
 synapse/handlers/search.py           | 10 ++++------
 synapse/rest/admin/rooms.py          | 12 +++++++++---
 synapse/rest/client/events.py        |  4 +---
 synapse/rest/client/notifications.py |  1 -
 synapse/rest/client/relations.py     |  4 +++-
 synapse/rest/client/room.py          | 10 ++++++----
 11 files changed, 35 insertions(+), 37 deletions(-)

diff --git a/synapse/events/utils.py b/synapse/events/utils.py
index 3da432c1df58..2038e72924a9 100644
--- a/synapse/events/utils.py
+++ b/synapse/events/utils.py
@@ -395,7 +395,7 @@ async def serialize_event(
         event: Union[JsonDict, EventBase],
         time_now: int,
         *,
-        bundle_aggregations: bool = True,
+        bundle_aggregations: bool = False,
         **kwargs: Any,
     ) -> JsonDict:
         """Serializes a single event.
diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py
index afed80ba14c0..1b996c420d20 100644
--- a/synapse/handlers/events.py
+++ b/synapse/handlers/events.py
@@ -123,8 +123,6 @@ async def get_stream(
                 events,
                 time_now,
                 as_client_event=as_client_event,
-                # Don't bundle aggregations as this is a deprecated API.
-                bundle_aggregations=False,
             )
 
             chunk = {
diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py
index 9ab723ff975e..601bab67f9cc 100644
--- a/synapse/handlers/initial_sync.py
+++ b/synapse/handlers/initial_sync.py
@@ -173,8 +173,6 @@ async def handle_room(event: RoomsForUser) -> None:
                 d["invite"] = await self._event_serializer.serialize_event(
                     invite_event,
                     time_now,
-                    # Don't bundle aggregations as this is a deprecated API.
-                    bundle_aggregations=False,
                     as_client_event=as_client_event,
                 )
 
@@ -227,8 +225,6 @@ async def handle_room(event: RoomsForUser) -> None:
                         await self._event_serializer.serialize_events(
                             messages,
                             time_now=time_now,
-                            # Don't bundle aggregations as this is a deprecated API.
-                            bundle_aggregations=False,
                             as_client_event=as_client_event,
                         )
                     ),
@@ -239,8 +235,6 @@ async def handle_room(event: RoomsForUser) -> None:
                 d["state"] = await self._event_serializer.serialize_events(
                     current_state.values(),
                     time_now=time_now,
-                    # Don't bundle aggregations as this is a deprecated API.
-                    bundle_aggregations=False,
                     as_client_event=as_client_event,
                 )
 
@@ -382,9 +376,7 @@ async def _room_initial_sync_parted(
             "messages": {
                 "chunk": (
                     # Don't bundle aggregations as this is a deprecated API.
-                    await self._event_serializer.serialize_events(
-                        messages, time_now, bundle_aggregations=False
-                    )
+                    await self._event_serializer.serialize_events(messages, time_now)
                 ),
                 "start": await start_token.to_string(self.store),
                 "end": await end_token.to_string(self.store),
@@ -392,7 +384,7 @@ async def _room_initial_sync_parted(
             "state": (
                 # Don't bundle aggregations as this is a deprecated API.
                 await self._event_serializer.serialize_events(
-                    room_state.values(), time_now, bundle_aggregations=False
+                    room_state.values(), time_now
                 )
             ),
             "presence": [],
@@ -413,7 +405,7 @@ async def _room_initial_sync_joined(
         time_now = self.clock.time_msec()
         # Don't bundle aggregations as this is a deprecated API.
         state = await self._event_serializer.serialize_events(
-            current_state.values(), time_now, bundle_aggregations=False
+            current_state.values(), time_now
         )
 
         now_token = self.hs.get_event_sources().get_current_token()
@@ -488,9 +480,7 @@ async def get_receipts() -> List[JsonDict]:
             "messages": {
                 "chunk": (
                     # Don't bundle aggregations as this is a deprecated API.
-                    await self._event_serializer.serialize_events(
-                        messages, time_now, bundle_aggregations=False
-                    )
+                    await self._event_serializer.serialize_events(messages, time_now)
                 ),
                 "start": await start_token.to_string(self.store),
                 "end": await end_token.to_string(self.store),
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 5e3d3886eb1d..1a7190085a7d 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -246,7 +246,9 @@ async def get_state_events(
                 room_state = room_state_events[membership_event_id]
 
         now = self.clock.time_msec()
-        events = await self._event_serializer.serialize_events(room_state.values(), now)
+        events = await self._event_serializer.serialize_events(
+            room_state.values(), now, bundle_aggregations=True
+        )
         return events
 
     async def get_joined_members(self, requester: Requester, room_id: str) -> dict:
diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py
index 4f424380533b..7469cc55a251 100644
--- a/synapse/handlers/pagination.py
+++ b/synapse/handlers/pagination.py
@@ -542,7 +542,10 @@ async def get_messages(
         chunk = {
             "chunk": (
                 await self._event_serializer.serialize_events(
-                    events, time_now, as_client_event=as_client_event
+                    events,
+                    time_now,
+                    bundle_aggregations=True,
+                    as_client_event=as_client_event,
                 )
             ),
             "start": await from_token.to_string(self.store),
diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py
index 1624abc07866..ab7eaab2fb56 100644
--- a/synapse/handlers/search.py
+++ b/synapse/handlers/search.py
@@ -421,10 +421,10 @@ async def search(
 
         for context in contexts.values():
             context["events_before"] = await self._event_serializer.serialize_events(
-                context["events_before"], time_now, bundle_aggregations=False
+                context["events_before"], time_now
             )
             context["events_after"] = await self._event_serializer.serialize_events(
-                context["events_after"], time_now, bundle_aggregations=False
+                context["events_after"], time_now
             )
 
         state_results = {}
@@ -442,9 +442,7 @@ async def search(
                 {
                     "rank": rank_map[e.event_id],
                     "result": (
-                        await self._event_serializer.serialize_event(
-                            e, time_now, bundle_aggregations=False
-                        )
+                        await self._event_serializer.serialize_event(e, time_now)
                     ),
                     "context": contexts.get(e.event_id, {}),
                 }
@@ -460,7 +458,7 @@ async def search(
             s = {}
             for room_id, state_events in state_results.items():
                 s[room_id] = await self._event_serializer.serialize_events(
-                    state_events, time_now, bundle_aggregations=False
+                    state_events, time_now
                 )
 
             rooms_cat_res["state"] = s
diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py
index 17c6df1cc8c7..6030373ebcbb 100644
--- a/synapse/rest/admin/rooms.py
+++ b/synapse/rest/admin/rooms.py
@@ -745,13 +745,19 @@ async def on_GET(
 
         time_now = self.clock.time_msec()
         results["events_before"] = await self._event_serializer.serialize_events(
-            results["events_before"], time_now
+            results["events_before"],
+            time_now,
+            bundle_aggregations=True,
         )
         results["event"] = await self._event_serializer.serialize_event(
-            results["event"], time_now
+            results["event"],
+            time_now,
+            bundle_aggregations=True,
         )
         results["events_after"] = await self._event_serializer.serialize_events(
-            results["events_after"], time_now
+            results["events_after"],
+            time_now,
+            bundle_aggregations=True,
         )
         results["state"] = await self._event_serializer.serialize_events(
             results["state"], time_now
diff --git a/synapse/rest/client/events.py b/synapse/rest/client/events.py
index cd4ff616a1e9..13b72a045a4a 100644
--- a/synapse/rest/client/events.py
+++ b/synapse/rest/client/events.py
@@ -91,9 +91,7 @@ async def on_GET(
 
         time_now = self.clock.time_msec()
         if event:
-            result = await self._event_serializer.serialize_event(
-                event, time_now, bundle_aggregations=False
-            )
+            result = await self._event_serializer.serialize_event(event, time_now)
             return 200, result
         else:
             return 404, "Event not found."
diff --git a/synapse/rest/client/notifications.py b/synapse/rest/client/notifications.py
index 1de2dd6f24ab..b12a332776e4 100644
--- a/synapse/rest/client/notifications.py
+++ b/synapse/rest/client/notifications.py
@@ -75,7 +75,6 @@ async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
                     await self._event_serializer.serialize_event(
                         notif_events[pa["event_id"]],
                         self.clock.time_msec(),
-                        bundle_aggregations=False,
                         event_format=format_event_for_client_v2_without_room_id,
                     )
                 ),
diff --git a/synapse/rest/client/relations.py b/synapse/rest/client/relations.py
index ffa37ef06c89..5815650ee6cb 100644
--- a/synapse/rest/client/relations.py
+++ b/synapse/rest/client/relations.py
@@ -232,7 +232,9 @@ async def on_GET(
         )
         # The relations returned for the requested event do include their
         # bundled aggregations.
-        serialized_events = await self._event_serializer.serialize_events(events, now)
+        serialized_events = await self._event_serializer.serialize_events(
+            events, now, bundle_aggregations=True
+        )
 
         return_value = pagination_chunk.to_dict()
         return_value["chunk"] = serialized_events
diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py
index 60719331b640..40330749e546 100644
--- a/synapse/rest/client/room.py
+++ b/synapse/rest/client/room.py
@@ -662,7 +662,9 @@ async def on_GET(
 
         time_now = self.clock.time_msec()
         if event:
-            event_dict = await self._event_serializer.serialize_event(event, time_now)
+            event_dict = await self._event_serializer.serialize_event(
+                event, time_now, bundle_aggregations=True
+            )
             return 200, event_dict
 
         raise SynapseError(404, "Event not found.", errcode=Codes.NOT_FOUND)
@@ -707,13 +709,13 @@ async def on_GET(
 
         time_now = self.clock.time_msec()
         results["events_before"] = await self._event_serializer.serialize_events(
-            results["events_before"], time_now
+            results["events_before"], time_now, bundle_aggregations=True
         )
         results["event"] = await self._event_serializer.serialize_event(
-            results["event"], time_now
+            results["event"], time_now, bundle_aggregations=True
         )
         results["events_after"] = await self._event_serializer.serialize_events(
-            results["events_after"], time_now
+            results["events_after"], time_now, bundle_aggregations=True
         )
         results["state"] = await self._event_serializer.serialize_events(
             results["state"], time_now