Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Lower minumum batch size to 1 for background updates #11422

Merged
merged 5 commits into from
Nov 24, 2021
Merged

Lower minumum batch size to 1 for background updates #11422

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1a8f5fd
Lower minumum batch size to 1
babolivier Nov 24, 2021
0c97b81
Changelog
babolivier Nov 24, 2021
7dfaf00
Fix tests
babolivier Nov 24, 2021
8bed262
Use larger values to compensate for the lower batch size
babolivier Nov 24, 2021
21b93ad
Update changelog.d/11422.bugfix
babolivier Nov 24, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/11422.bugfix
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Improve performance of various background database schema updates.
2 changes: 1 addition & 1 deletion synapse/storage/background_updates.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ class BackgroundUpdater:
process and autotuning the batch size.
"""

MINIMUM_BACKGROUND_BATCH_SIZE = 100
MINIMUM_BACKGROUND_BATCH_SIZE = 1
DEFAULT_BACKGROUND_BATCH_SIZE = 100
BACKGROUND_UPDATE_INTERVAL_MS = 1000
BACKGROUND_UPDATE_DURATION_MS = 100
Expand Down
25 changes: 17 additions & 8 deletions tests/rest/admin/test_background_updates.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from synapse.api.errors import Codes
from synapse.rest.client import login
from synapse.server import HomeServer
from synapse.storage.background_updates import BackgroundUpdater

from tests import unittest

Expand Down Expand Up @@ -150,9 +151,11 @@ def test_status_bg_update(self):
"current_updates": {
"master": {
"name": "test_update",
"average_items_per_ms": 0.1,
"average_items_per_ms": 0.001,
"total_duration_ms": 1000.0,
"total_item_count": 100,
"total_item_count": (
BackgroundUpdater.MINIMUM_BACKGROUND_BATCH_SIZE
),
}
},
"enabled": True,
Expand Down Expand Up @@ -203,9 +206,11 @@ def test_enabled(self):
"current_updates": {
"master": {
"name": "test_update",
"average_items_per_ms": 0.1,
"average_items_per_ms": 0.001,
"total_duration_ms": 1000.0,
"total_item_count": 100,
"total_item_count": (
BackgroundUpdater.MINIMUM_BACKGROUND_BATCH_SIZE
),
}
},
"enabled": False,
Expand All @@ -230,9 +235,11 @@ def test_enabled(self):
"current_updates": {
"master": {
"name": "test_update",
"average_items_per_ms": 0.1,
"average_items_per_ms": 0.001,
"total_duration_ms": 1000.0,
"total_item_count": 100,
"total_item_count": (
BackgroundUpdater.MINIMUM_BACKGROUND_BATCH_SIZE
),
}
},
"enabled": False,
Expand Down Expand Up @@ -267,9 +274,11 @@ def test_enabled(self):
"current_updates": {
"master": {
"name": "test_update",
"average_items_per_ms": 0.1,
"average_items_per_ms": 0.001,
"total_duration_ms": 2000.0,
"total_item_count": 200,
"total_item_count": (
2 * BackgroundUpdater.MINIMUM_BACKGROUND_BATCH_SIZE
),
}
},
"enabled": True,
Expand Down
8 changes: 4 additions & 4 deletions tests/storage/test_background_update.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,11 @@ def prepare(self, reactor, clock, homeserver):
)

def test_do_background_update(self):
# the time we claim each update takes
duration_ms = 42
# the time we claim it takes to update one item when running the update
duration_ms = 4200

# the target runtime for each bg update
target_background_update_duration_ms = 50000
target_background_update_duration_ms = 5000000

store = self.hs.get_datastore()
self.get_success(
Expand Down Expand Up @@ -57,7 +57,7 @@ async def update(progress, count):

# on the first call, we should get run with the default background update size
self.update_handler.assert_called_once_with(
{"my_key": 1}, self.updates.DEFAULT_BACKGROUND_BATCH_SIZE
{"my_key": 1}, self.updates.MINIMUM_BACKGROUND_BATCH_SIZE
)

# second step: complete the update
Expand Down