From b2551de5362177aff77d152f9303791225c08288 Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Fri, 1 Nov 2013 11:49:15 -0700 Subject: [PATCH 1/2] Use region specific endpoint when updating s3 hostname This fixes an issue when using govcloud regions, as `s3.amazonaws.com` won't map to the govcloud region. All botocore/awscli unit/integ tests pass --- botocore/handlers.py | 2 +- tests/unit/test_s3_addressing.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/botocore/handlers.py b/botocore/handlers.py index 4c121620c0..93bbfe4345 100644 --- a/botocore/handlers.py +++ b/botocore/handlers.py @@ -121,7 +121,7 @@ def fix_s3_host(event_name, endpoint, request, auth, **kwargs): if auth.auth_path[-1] != '/': auth.auth_path += '/' path_parts.remove(bucket_name) - host = bucket_name + '.' + endpoint.service.global_endpoint + host = bucket_name + '.' + parts.netloc new_tuple = (parts.scheme, host, '/'.join(path_parts), parts.query, '') new_uri = urlunsplit(new_tuple) diff --git a/tests/unit/test_s3_addressing.py b/tests/unit/test_s3_addressing.py index 26be381723..0fd5e0e77a 100644 --- a/tests/unit/test_s3_addressing.py +++ b/tests/unit/test_s3_addressing.py @@ -67,8 +67,9 @@ def test_list_objects_dns_name_non_classic(self): op = self.s3.get_operation('ListObjects') params = op.build_parameters(bucket='safename') prepared_request = self.get_prepared_request(op, params) + # Note how we keep the region specific endpoint here. self.assertEqual(prepared_request.url, - 'https://safename.s3.amazonaws.com/') + 'https://safename.s3-us-west-2.amazonaws.com/') def test_list_objects_non_dns_name_non_classic(self): self.endpoint = self.s3.get_endpoint('us-west-2') From be4ed78874f8f5f6fe67f17fc6740f9ab4e10406 Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Fri, 1 Nov 2013 17:02:39 -0700 Subject: [PATCH 2/2] Keep a list of restricted regions that aren't remapped Fixes an issue when using the govcloud region for s3. --- botocore/handlers.py | 17 +++++++++++++---- tests/unit/test_s3_addressing.py | 10 +++++++++- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/botocore/handlers.py b/botocore/handlers.py index 93bbfe4345..418381afe0 100644 --- a/botocore/handlers.py +++ b/botocore/handlers.py @@ -36,7 +36,12 @@ logger = logging.getLogger(__name__) -LabelRE = re.compile('[a-z0-9][a-z0-9\-]*[a-z0-9]') +LABEL_RE = re.compile('[a-z0-9][a-z0-9\-]*[a-z0-9]') +RESTRICTED_REGIONS = [ + 'us-gov-west-1', + 'fips-gov-west-1', +] + def decode_console_output(event_name, shape, value, **kwargs): @@ -91,7 +96,7 @@ def check_dns_name(bucket_name): if n == 1: if not bucket_name.isalnum(): return False - match = LabelRE.match(bucket_name) + match = LABEL_RE.match(bucket_name) if match is None or match.end() != len(bucket_name): return False return True @@ -114,14 +119,14 @@ def fix_s3_host(event_name, endpoint, request, auth, **kwargs): bucket_name = path_parts[1] logger.debug('Checking for DNS compatible bucket for: %s', request.url) - if check_dns_name(bucket_name): + if check_dns_name(bucket_name) and _allowed_region(endpoint.region_name): # If the operation is on a bucket, the auth_path must be # terminated with a '/' character. if len(path_parts) == 2: if auth.auth_path[-1] != '/': auth.auth_path += '/' path_parts.remove(bucket_name) - host = bucket_name + '.' + parts.netloc + host = bucket_name + '.' + endpoint.service.global_endpoint new_tuple = (parts.scheme, host, '/'.join(path_parts), parts.query, '') new_uri = urlunsplit(new_tuple) @@ -132,6 +137,10 @@ def fix_s3_host(event_name, endpoint, request, auth, **kwargs): bucket_name) +def _allowed_region(region_name): + return region_name not in RESTRICTED_REGIONS + + def register_retries_for_service(service, **kwargs): if not hasattr(service, 'retry'): return diff --git a/tests/unit/test_s3_addressing.py b/tests/unit/test_s3_addressing.py index 0fd5e0e77a..cee2b7cdba 100644 --- a/tests/unit/test_s3_addressing.py +++ b/tests/unit/test_s3_addressing.py @@ -67,9 +67,17 @@ def test_list_objects_dns_name_non_classic(self): op = self.s3.get_operation('ListObjects') params = op.build_parameters(bucket='safename') prepared_request = self.get_prepared_request(op, params) + self.assertEqual(prepared_request.url, + 'https://safename.s3.amazonaws.com/') + + def test_list_objects_in_restricted_regions(self): + self.endpoint = self.s3.get_endpoint('us-gov-west-1') + op = self.s3.get_operation('ListObjects') + params = op.build_parameters(bucket='safename') + prepared_request = self.get_prepared_request(op, params) # Note how we keep the region specific endpoint here. self.assertEqual(prepared_request.url, - 'https://safename.s3-us-west-2.amazonaws.com/') + 'https://s3-us-gov-west-1.amazonaws.com/safename') def test_list_objects_non_dns_name_non_classic(self): self.endpoint = self.s3.get_endpoint('us-west-2')