Skip to content

Commit

Permalink
Merge pull request #9560 from rtibbles/multiprocess_download
Browse files Browse the repository at this point in the history
Don't use multiprocessing for downloads.
rtibbles authored Jul 12, 2022

Verified

This commit was signed with the committer’s verified signature.
mikz Michal Cichra
2 parents e4502b4 + 7ffc85b commit d150cb3
Showing 1 changed file with 7 additions and 2 deletions.
9 changes: 7 additions & 2 deletions kolibri/core/content/management/commands/importcontent.py
Original file line number Diff line number Diff line change
@@ -346,15 +346,20 @@ def _transfer( # noqa: max-complexity=16
if method == DOWNLOAD_METHOD:
session = requests.Session()

# We should be deferring to conf.OPTIONS["Tasks"]["USE_WORKER_MULTIPROCESSING"]
# for this value, but unfortunately, the current way that the import logic
# is setup relies on shared memory that can only be used with threads.
use_multiprocessing = False

executor = (
concurrent.futures.ProcessPoolExecutor
if conf.OPTIONS["Tasks"]["USE_WORKER_MULTIPROCESSING"]
if use_multiprocessing
else concurrent.futures.ThreadPoolExecutor
)

max_workers = 10

if not conf.OPTIONS["Tasks"]["USE_WORKER_MULTIPROCESSING"]:
if not use_multiprocessing:
# If we're not using multiprocessing for workers, we may need
# to limit the number of workers depending on the number of allowed
# file descriptors.

0 comments on commit d150cb3

Please sign in to comment.