Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[v2] Expose --bucket-name-prefix and --bucket-region to s3 ls #9163

Merged
merged 5 commits into from
Jan 6, 2025
Merged

[v2] Expose --bucket-name-prefix and --bucket-region to s3 ls #9163

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
16a4898
Expose bucket-name-prefix and bucket-region params to s3 ls
hssyoo Dec 23, 2024
5c12589
Write tests
hssyoo Dec 23, 2024
4f62b8a
Add changelog
hssyoo Dec 23, 2024
660b939
Fix formatting
hssyoo Dec 23, 2024
abc78a4
Add tests to validate ListObjectsV2 ignores new params
hssyoo Jan 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changes/next-release/enhancement-s3ls-20704.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"type": "enhancement",
"category": "``s3 ls``",
"description": "Expose low-level ``ListBuckets` parameters ``Prefix`` and ``BucketRegion`` to high-level ``s3 ls`` command as ``--bucket-name-prefix`` and ``--bucket-region``."
}
40 changes: 37 additions & 3 deletions awscli/customizations/s3/subcommands.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -492,6 +492,26 @@
'help_text': 'Indicates the algorithm used to create the checksum for the object.'
}

BUCKET_NAME_PREFIX = {
'name': 'bucket-name-prefix',
'help_text': (
'Limits the response to bucket names that begin with the specified '
'bucket name prefix.'
)
}

BUCKET_REGION = {
'name': 'bucket-region',
'help_text': (
'Limits the response to buckets that are located in the specified '
'Amazon Web Services Region. The Amazon Web Services Region must be '
'expressed according to the Amazon Web Services Region code, such as '
'us-west-2 for the US West (Oregon) Region. For a list of the valid '
'values for all of the Amazon Web Services Regions, see '
'https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region'
)
}

TRANSFER_ARGS = [DRYRUN, QUIET, INCLUDE, EXCLUDE, ACL,
FOLLOW_SYMLINKS, NO_FOLLOW_SYMLINKS, NO_GUESS_MIME_TYPE,
SSE, SSE_C, SSE_C_KEY, SSE_KMS_KEY_ID, SSE_C_COPY_SOURCE,
Expand Down Expand Up @@ -521,7 +541,8 @@ class ListCommand(S3Command):
USAGE = "<S3Uri> or NONE"
ARG_TABLE = [{'name': 'paths', 'nargs': '?', 'default': 's3://',
'positional_arg': True, 'synopsis': USAGE}, RECURSIVE,
PAGE_SIZE, HUMAN_READABLE, SUMMARIZE, REQUEST_PAYER]
PAGE_SIZE, HUMAN_READABLE, SUMMARIZE, REQUEST_PAYER,
BUCKET_NAME_PREFIX, BUCKET_REGION]

def _run_main(self, parsed_args, parsed_globals):
super(ListCommand, self)._run_main(parsed_args, parsed_globals)
Expand All @@ -535,7 +556,11 @@ def _run_main(self, parsed_args, parsed_globals):
path = path[5:]
bucket, key = find_bucket_key(path)
if not bucket:
self._list_all_buckets(parsed_args.page_size)
self._list_all_buckets(
parsed_args.page_size,
parsed_args.bucket_name_prefix,
parsed_args.bucket_region,
)
elif parsed_args.dir_op:
# Then --recursive was specified.
self._list_all_objects_recursive(
Expand Down Expand Up @@ -599,11 +624,20 @@ def _display_page(self, response_data, use_basename=True):
uni_print(print_str)
self._at_first_page = False

def _list_all_buckets(self, page_size=None):
def _list_all_buckets(
self,
page_size=None,
prefix=None,
bucket_region=None,
):
paginator = self.client.get_paginator('list_buckets')
paging_args = {
'PaginationConfig': {'PageSize': page_size}
}
if prefix:
paging_args['Prefix'] = prefix
if bucket_region:
paging_args['BucketRegion'] = bucket_region

iterator = paginator.paginate(**paging_args)

Expand Down
20 changes: 20 additions & 0 deletions tests/functional/s3/test_ls_command.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,3 +215,23 @@ def test_accesspoint_arn(self):
self.run_cmd('s3 ls s3://%s' % arn, expected_rc=0)
call_args = self.operations_called[0][1]
self.assertEqual(call_args['Bucket'], arn)

def test_list_buckets_uses_bucket_name_prefix(self):
stdout, _, _ = self.run_cmd('s3 ls --bucket-name-prefix myprefix', expected_rc=0)
call_args = self.operations_called[0][1]
self.assertEqual(call_args['Prefix'], 'myprefix')

def test_list_buckets_uses_bucket_region(self):
stdout, _, _ = self.run_cmd('s3 ls --bucket-region us-west-1', expected_rc=0)
call_args = self.operations_called[0][1]
self.assertEqual(call_args['BucketRegion'], 'us-west-1')

def test_list_objects_ignores_bucket_name_prefix(self):
stdout, _, _ = self.run_cmd('s3 ls s3://mybucket --bucket-name-prefix myprefix', expected_rc=0)
call_args = self.operations_called[0][1]
self.assertEqual(call_args['Prefix'], '')

def test_list_objects_ignores_bucket_region(self):
stdout, _, _ = self.run_cmd('s3 ls s3://mybucket --bucket-region us-west-1', expected_rc=0)
call_args = self.operations_called[0][1]
self.assertNotIn('BucketRegion', call_args)
89 changes: 76 additions & 13 deletions tests/unit/customizations/s3/test_subcommands.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,11 +79,27 @@ def setUp(self):
self.session.create_client.return_value.get_paginator.return_value\
.paginate.return_value = [{'Contents': [], 'CommonPrefixes': []}]

def _get_fake_kwargs(self, override=None):
fake_kwargs = {
'paths': 's3://',
'dir_op': False,
'human_readable': False,
'summarize': False,
'page_size': None,
'request_payer': None,
'bucket_name_prefix': None,
'bucket_region': None,
}
fake_kwargs.update(override or {})

return fake_kwargs

def test_ls_command_for_bucket(self):
ls_command = ListCommand(self.session)
parsed_args = FakeArgs(paths='s3://mybucket/', dir_op=False,
page_size='5', human_readable=False,
summarize=False, request_payer=None)
parsed_args = FakeArgs(**self._get_fake_kwargs({
'paths': 's3://mybucket/',
'page_size': '5',
}))
parsed_globals = mock.Mock()
ls_command._run_main(parsed_args, parsed_globals)
call = self.session.create_client.return_value.list_objects_v2
Expand All @@ -104,9 +120,7 @@ def test_ls_command_with_no_args(self):
ls_command = ListCommand(self.session)
parsed_global = FakeArgs(region=None, endpoint_url=None,
verify_ssl=None)
parsed_args = FakeArgs(dir_op=False, paths='s3://',
human_readable=False, summarize=False,
request_payer=None, page_size=None)
parsed_args = FakeArgs(**self._get_fake_kwargs())
ls_command._run_main(parsed_args, parsed_global)
call = self.session.create_client.return_value.list_buckets
paginate = self.session.create_client.return_value.get_paginator\
Expand All @@ -129,14 +143,61 @@ def test_ls_command_with_no_args(self):
mock.call('s3', region_name=None, verify=None, endpoint_url=None)
)

def test_ls_with_bucket_name_prefix(self):
ls_command = ListCommand(self.session)
parsed_args = FakeArgs(**self._get_fake_kwargs({
'bucket_name_prefix': 'myprefix',
}))
parsed_globals = FakeArgs(
region=None,
endpoint_url=None,
verify_ssl=None,
)
ls_command._run_main(parsed_args, parsed_globals)
call = self.session.create_client.return_value.list_objects
paginate = self.session.create_client.return_value.get_paginator\
.return_value.paginate
# We should make no operation calls.
self.assertEqual(call.call_count, 0)
self.session.create_client.return_value.get_paginator.\
assert_called_with('list_buckets')
ref_call_args = {
'PaginationConfig': {'PageSize': None},
'Prefix': 'myprefix',
}

paginate.assert_called_with(**ref_call_args)

def test_ls_with_bucket_region(self):
ls_command = ListCommand(self.session)
parsed_args = FakeArgs(**self._get_fake_kwargs({
'bucket_region': 'us-west-1',
}))
parsed_globals = FakeArgs(
region=None,
endpoint_url=None,
verify_ssl=None,
)
ls_command._run_main(parsed_args, parsed_globals)
call = self.session.create_client.return_value.list_objects
paginate = self.session.create_client.return_value.get_paginator\
.return_value.paginate
# We should make no operation calls.
self.assertEqual(call.call_count, 0)
self.session.create_client.return_value.get_paginator.\
assert_called_with('list_buckets')
ref_call_args = {
'PaginationConfig': {'PageSize': None},
'BucketRegion': 'us-west-1',
}

paginate.assert_called_with(**ref_call_args)

def test_ls_with_verify_argument(self):
options = {'default': 's3://', 'nargs': '?'}
ls_command = ListCommand(self.session)
parsed_global = FakeArgs(region='us-west-2', endpoint_url=None,
verify_ssl=False)
parsed_args = FakeArgs(paths='s3://', dir_op=False,
human_readable=False, summarize=False,
request_payer=None, page_size=None)
parsed_args = FakeArgs(**self._get_fake_kwargs({}))
ls_command._run_main(parsed_args, parsed_global)
# Verify get_client
get_client = self.session.create_client
Expand All @@ -150,9 +211,11 @@ def test_ls_with_verify_argument(self):

def test_ls_with_requester_pays(self):
ls_command = ListCommand(self.session)
parsed_args = FakeArgs(paths='s3://mybucket/', dir_op=False,
human_readable=False, summarize=False,
request_payer='requester', page_size='5')
parsed_args = FakeArgs(**self._get_fake_kwargs({
'paths': 's3://mybucket/',
'page_size': '5',
'request_payer': 'requester',
}))
parsed_globals = mock.Mock()
ls_command._run_main(parsed_args, parsed_globals)
call = self.session.create_client.return_value.list_objects
Expand Down
Loading