From a21fed797300fd4911588785a690f9ba24a99b03 Mon Sep 17 00:00:00 2001 From: Tom Savage Date: Thu, 8 Nov 2018 14:39:48 +0000 Subject: [PATCH 1/3] fix: Use continuation token when requesting a page of keys if set The continuation token is being captured when each page of keys is fetched, but it was not used for subsequent requests. This meant that the same page of 1000 keys would repeat forever. This commit adds a fix to ensure the continuation token is used if not None --- smart_open/s3.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/smart_open/s3.py b/smart_open/s3.py index 0650c2ab..867614c9 100644 --- a/smart_open/s3.py +++ b/smart_open/s3.py @@ -535,7 +535,12 @@ def _list_bucket(bucket_name, prefix='', accept_key=lambda k: True): ctoken = None while True: - response = client.list_objects_v2(Bucket=bucket_name, Prefix=prefix) + # list_objects_v2 doesn't like a None value for ContinuationToken + # so we don't set if if we don't have one. + if ctoken: + response = client.list_objects_v2(Bucket=bucket_name, Prefix=prefix, ContinuationToken=ctoken) + else: + response = client.list_objects_v2(Bucket=bucket_name, Prefix=prefix) try: content = response['Contents'] except KeyError: From 4417a0f1a541eac7ae5e950e706026de25d49c43 Mon Sep 17 00:00:00 2001 From: Michael Penkov Date: Fri, 9 Nov 2018 09:53:14 +0000 Subject: [PATCH 2/3] Fix typo in comment spotted by mpenkov Co-Authored-By: tcsavage --- smart_open/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smart_open/s3.py b/smart_open/s3.py index 867614c9..1bd7eec2 100644 --- a/smart_open/s3.py +++ b/smart_open/s3.py @@ -536,7 +536,7 @@ def _list_bucket(bucket_name, prefix='', accept_key=lambda k: True): while True: # list_objects_v2 doesn't like a None value for ContinuationToken - # so we don't set if if we don't have one. + # so we don't set it if we don't have one. if ctoken: response = client.list_objects_v2(Bucket=bucket_name, Prefix=prefix, ContinuationToken=ctoken) else: From f0d22fea3f6f1358666f3f9cfe2a62191d003671 Mon Sep 17 00:00:00 2001 From: Tom Savage Date: Fri, 9 Nov 2018 09:56:19 +0000 Subject: [PATCH 3/3] test: Remove skip decorator for test_list_bucket_long --- smart_open/tests/test_s3.py | 1 - 1 file changed, 1 deletion(-) diff --git a/smart_open/tests/test_s3.py b/smart_open/tests/test_s3.py index fa02b9c5..237a5f26 100644 --- a/smart_open/tests/test_s3.py +++ b/smart_open/tests/test_s3.py @@ -391,7 +391,6 @@ def test_list_bucket(self): expected = ['key_%d' % x for x in range(num_keys)] self.assertEqual(sorted(keys), sorted(expected)) - @unittest.skip('this test takes too long for some unknown reason') def test_list_bucket_long(self): num_keys = 1010 populate_bucket(num_keys=num_keys)