From 11b2d9298562afa72e292989cd6a7be409cf15f6 Mon Sep 17 00:00:00 2001 From: Alex Tomkins Date: Thu, 27 Jul 2017 20:16:20 +0100 Subject: [PATCH] Create a session/connect per thread for s3boto3 (#358) Documentation for boto3 recommends a session per thread - https://boto3.readthedocs.io/en/latest/guide/resources.html#multithreading-multiprocessing As the storage class is (usually) only instantiated once per process, we need to set a thread local for each thread/connection used. Fixes #268 --- storages/backends/s3boto3.py | 10 ++++++---- tests/test_s3boto3.py | 23 ++++++++++++++++++++--- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/storages/backends/s3boto3.py b/storages/backends/s3boto3.py index 1547a47de..6c18ced60 100644 --- a/storages/backends/s3boto3.py +++ b/storages/backends/s3boto3.py @@ -1,6 +1,7 @@ import mimetypes import os import posixpath +import threading from gzip import GzipFile from tempfile import SpooledTemporaryFile @@ -236,7 +237,7 @@ def __init__(self, acl=None, bucket=None, **settings): self._entries = {} self._bucket = None - self._connection = None + self._connections = threading.local() self.security_token = None if not self.access_key and not self.secret_key: @@ -253,9 +254,10 @@ def connection(self): # Note that proxies are handled by environment variables that the underlying # urllib/requests libraries read. See https://github.com/boto/boto3/issues/338 # and http://docs.python-requests.org/en/latest/user/advanced/#proxies - if self._connection is None: + connection = getattr(self._connections, 'connection', None) + if connection is None: session = boto3.session.Session() - self._connection = session.resource( + self._connections.connection = session.resource( 's3', aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key, @@ -265,7 +267,7 @@ def connection(self): endpoint_url=self.endpoint_url, config=self.config ) - return self._connection + return self._connections.connection @property def bucket(self): diff --git a/tests/test_s3boto3.py b/tests/test_s3boto3.py index adfed69b5..d0f0b0593 100644 --- a/tests/test_s3boto3.py +++ b/tests/test_s3boto3.py @@ -2,7 +2,9 @@ from __future__ import unicode_literals import gzip +import threading from datetime import datetime +from unittest import skipIf from botocore.exceptions import ClientError from django.conf import settings @@ -22,7 +24,7 @@ class S3Boto3TestCase(TestCase): def setUp(self): self.storage = s3boto3.S3Boto3Storage() - self.storage._connection = mock.MagicMock() + self.storage._connections.connection = mock.MagicMock() class S3Boto3StorageTests(S3Boto3TestCase): @@ -174,8 +176,8 @@ def test_storage_open_write(self): def test_auto_creating_bucket(self): self.storage.auto_create_bucket = True Bucket = mock.MagicMock() - self.storage._connection.Bucket.return_value = Bucket - self.storage._connection.meta.client.meta.region_name = 'sa-east-1' + self.storage._connections.connection.Bucket.return_value = Bucket + self.storage._connections.connection.meta.client.meta.region_name = 'sa-east-1' Bucket.meta.client.head_bucket.side_effect = ClientError({'Error': {}, 'ResponseMetadata': {'HTTPStatusCode': 404}}, @@ -342,3 +344,18 @@ def test_strip_signing_parameters(self): '%s?X-Amz-Date=12345678&X-Amz-Signature=Signature' % expected), expected) self.assertEqual(self.storage._strip_signing_parameters( '%s?expires=12345678&signature=Signature' % expected), expected) + + @skipIf(threading is None, 'Test requires threading') + def test_connection_threading(self): + connections = [] + + def thread_storage_connection(): + connections.append(self.storage.connection) + + for x in range(2): + t = threading.Thread(target=thread_storage_connection) + t.start() + t.join() + + # Connection for each thread needs to be unique + self.assertIsNot(connections[0], connections[1])