From d258e03c399536f61ac539d4a5bcb5c2cd9eae87 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 6 Oct 2022 02:04:22 -0500 Subject: [PATCH 1/3] De-flake MSC3030 backfill test with Synapse worker replication Spawning from https://github.com/matrix-org/synapse/pull/14028#discussion_r987156197 The reason they failed before was because we're fighting against stale caches across workers racing while waiting for invalidation, see https://github.com/matrix-org/synapse/issues/13185#issuecomment-1269413897 > Here is what happens: > > 1. `serverB` has `event1` stored as an `outlier` from previous requests (specifically from MSC3030 jump to date pulling in a missing `prev_event` after backfilling) > 1. Client on `serverB` calls `/messages?dir=b` > 1. `serverB:client_reader1` accepts the request and drives things > 1. `serverB:client_reader1` has some backward extremities in range and requests `/backfill` from `serverA` > 1. `serverB:client_reader1` processes the events from backfill including `event1` and puts them in the `_event_persist_queue` > 1. `serverB:master` picks up the events from the `_event_persist_queue` and persists them to the database, de-outliers `event1` and invalidates its own cache and sends them over replication > 1. `serverB:client_reader1` starts assembling the `/messages` response and gets `event1` out of the stale cache still as an `outlier` > 1. `serverB:client_reader1` responds to the `/messages` request without `event1` because `outliers` are filtered out > 1. `serverB:client_reader1` finally gets the replication data and invalidates its own cache for `event1` (too late, we already got the events from the stale cache and responded) --- tests/msc3030_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/msc3030_test.go b/tests/msc3030_test.go index 061a63ad..5d32d4c2 100644 --- a/tests/msc3030_test.go +++ b/tests/msc3030_test.go @@ -203,6 +203,16 @@ func TestJumpToDateEndpoint(t *testing.T) { contextResResBody := client.ParseJSON(t, contextRes) paginationToken := client.GetJSONFieldStr(t, contextResResBody, "end") + // Hit `/messages` until `eventA` has been backfilled and replicated across + // workers. + fetchUntilMessagesResponseHas(t, remoteCharlie, roomID, func(ev gjson.Result) bool { + if ev.Get("event_id").Str == eventA.EventID { + return true + } + + return false + }) + // Paginate backwards from eventB messagesRes := remoteCharlie.MustDoFunc(t, "GET", []string{"_matrix", "client", "r0", "rooms", roomID, "messages"}, client.WithContentType("application/json"), client.WithQueries(url.Values{ "dir": []string{"b"}, From 5aebe260e836cb6056babdb06cbf5fb5812caa31 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 11 Oct 2022 13:17:34 -0500 Subject: [PATCH 2/3] Extra description on persisting worker not the same as `/messages` worker See https://github.com/matrix-org/complement/pull/492#discussion_r991225713 --- tests/msc3030_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/msc3030_test.go b/tests/msc3030_test.go index 5d32d4c2..b461f7c8 100644 --- a/tests/msc3030_test.go +++ b/tests/msc3030_test.go @@ -204,7 +204,8 @@ func TestJumpToDateEndpoint(t *testing.T) { paginationToken := client.GetJSONFieldStr(t, contextResResBody, "end") // Hit `/messages` until `eventA` has been backfilled and replicated across - // workers. + // workers (the worker persisting events isn't necessarily the same as the worker + // serving `/messages`) fetchUntilMessagesResponseHas(t, remoteCharlie, roomID, func(ev gjson.Result) bool { if ev.Get("event_id").Str == eventA.EventID { return true From dcf74dbb918d612d7fd3265830802e125bf65f9e Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 12 Oct 2022 17:56:33 -0500 Subject: [PATCH 3/3] Return condition result directly See https://github.com/matrix-org/complement/pull/492#discussion_r993257765 --- tests/msc3030_test.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/msc3030_test.go b/tests/msc3030_test.go index b461f7c8..8350b192 100644 --- a/tests/msc3030_test.go +++ b/tests/msc3030_test.go @@ -207,11 +207,7 @@ func TestJumpToDateEndpoint(t *testing.T) { // workers (the worker persisting events isn't necessarily the same as the worker // serving `/messages`) fetchUntilMessagesResponseHas(t, remoteCharlie, roomID, func(ev gjson.Result) bool { - if ev.Get("event_id").Str == eventA.EventID { - return true - } - - return false + return ev.Get("event_id").Str == eventA.EventID }) // Paginate backwards from eventB