From 2e2617214f11f6fcd33eac876a46fcadfa7e4ab9 Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Tue, 12 Nov 2024 18:56:49 +0000 Subject: [PATCH 1/4] feat: IAM signBlob retries --- google/cloud/storage/_signing.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/google/cloud/storage/_signing.py b/google/cloud/storage/_signing.py index ecf110769..e92decad1 100644 --- a/google/cloud/storage/_signing.py +++ b/google/cloud/storage/_signing.py @@ -30,6 +30,7 @@ from google.cloud import _helpers from google.cloud.storage._helpers import _NOW from google.cloud.storage._helpers import _UTC +from google.cloud.storage.retry import DEFAULT_RETRY # `google.cloud.storage._signing.NOW` is deprecated. @@ -677,9 +678,16 @@ def _sign_message(message, access_token, service_account_email): "Content-type": "application/json", } body = json.dumps({"payload": base64.b64encode(message).decode("utf-8")}) - request = requests.Request() - response = request(url=url, method=method, body=body, headers=headers) + + def retriable_request(): + response = request(url=url, method=method, body=body, headers=headers) + return response + + # Apply the default retry object to the signBlob call. + retry = DEFAULT_RETRY + call = retry(retriable_request) + response = call() if response.status != http.client.OK: raise exceptions.TransportError( From a825e35993538781e123b04f37d7c6c6f3287b1c Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Sat, 16 Nov 2024 00:11:05 +0000 Subject: [PATCH 2/4] support universe domain and update tests --- google/cloud/storage/_signing.py | 22 +++++++++++++++------ google/cloud/storage/blob.py | 4 ++++ tests/system/conftest.py | 15 ++++++++++++++ tests/system/test__signing.py | 34 ++++++++++++++++++++++++++++++++ tests/unit/test_blob.py | 3 +++ 5 files changed, 72 insertions(+), 6 deletions(-) diff --git a/google/cloud/storage/_signing.py b/google/cloud/storage/_signing.py index e92decad1..9f47e1a6e 100644 --- a/google/cloud/storage/_signing.py +++ b/google/cloud/storage/_signing.py @@ -28,6 +28,7 @@ from google.auth import exceptions from google.auth.transport import requests from google.cloud import _helpers +from google.cloud.storage._helpers import _DEFAULT_UNIVERSE_DOMAIN from google.cloud.storage._helpers import _NOW from google.cloud.storage._helpers import _UTC from google.cloud.storage.retry import DEFAULT_RETRY @@ -272,6 +273,7 @@ def generate_signed_url_v2( query_parameters=None, service_account_email=None, access_token=None, + universe_domain=None, ): """Generate a V2 signed URL to provide query-string auth'n to a resource. @@ -385,7 +387,9 @@ def generate_signed_url_v2( # See https://github.com/googleapis/google-cloud-python/issues/922 # Set the right query parameters. if access_token and service_account_email: - signature = _sign_message(string_to_sign, access_token, service_account_email) + signature = _sign_message( + string_to_sign, access_token, service_account_email, universe_domain + ) signed_query_params = { "GoogleAccessId": service_account_email, "Expires": expiration_stamp, @@ -433,6 +437,7 @@ def generate_signed_url_v4( query_parameters=None, service_account_email=None, access_token=None, + universe_domain=None, _request_timestamp=None, # for testing only ): """Generate a V4 signed URL to provide query-string auth'n to a resource. @@ -624,7 +629,9 @@ def generate_signed_url_v4( string_to_sign = "\n".join(string_elements) if access_token and service_account_email: - signature = _sign_message(string_to_sign, access_token, service_account_email) + signature = _sign_message( + string_to_sign, access_token, service_account_email, universe_domain + ) signature_bytes = base64.b64decode(signature) signature = binascii.hexlify(signature_bytes).decode("ascii") else: @@ -648,7 +655,12 @@ def get_v4_now_dtstamps(): return timestamp, datestamp -def _sign_message(message, access_token, service_account_email): +def _sign_message( + message, + access_token, + service_account_email, + universe_domain=_DEFAULT_UNIVERSE_DOMAIN, +): """Signs a message. :type message: str @@ -670,9 +682,7 @@ def _sign_message(message, access_token, service_account_email): message = _helpers._to_bytes(message) method = "POST" - url = "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/{}:signBlob?alt=json".format( - service_account_email - ) + url = f"https://iamcredentials.{universe_domain}/v1/projects/-/serviceAccounts/{service_account_email}:signBlob?alt=json" headers = { "Authorization": "Bearer " + access_token, "Content-type": "application/json", diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index e474f1681..9b7746ffe 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -607,6 +607,9 @@ def generate_signed_url( client = self._require_client(client) # May be redundant, but that's ok. credentials = client._credentials + client = self._require_client(client) + universe_domain = client.universe_domain + if version == "v2": helper = generate_signed_url_v2 else: @@ -638,6 +641,7 @@ def generate_signed_url( query_parameters=query_parameters, service_account_email=service_account_email, access_token=access_token, + universe_domain=universe_domain, ) @create_trace_span(name="Storage.Blob.exists") diff --git a/tests/system/conftest.py b/tests/system/conftest.py index 4ec56176d..7c045fe36 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -384,3 +384,18 @@ def universe_domain_client( ) with contextlib.closing(ud_storage_client): yield ud_storage_client + + +@pytest.fixture(scope="function") +def universe_domain_iam_client( + test_universe_domain, test_universe_project_id, universe_domain_credential +): + from google.cloud import iam_credentials_v1 + + client_options = {"universe_domain": test_universe_domain} + iam_client = iam_credentials_v1.IAMCredentialsClient( + credentials=universe_domain_credential, + client_options=client_options, + ) + with contextlib.closing(iam_client): + yield iam_client diff --git a/tests/system/test__signing.py b/tests/system/test__signing.py index 8bcc46abc..f1b83f771 100644 --- a/tests/system/test__signing.py +++ b/tests/system/test__signing.py @@ -287,6 +287,40 @@ def test_create_signed_read_url_v4_w_access_token( ) +def test_create_signed_read_url_v4_w_access_token_universe_domain( + universe_domain_iam_client, + universe_domain_client, + test_universe_location, + test_universe_domain_credential, + buckets_to_delete, + no_mtls, +): + service_account_email = test_universe_domain_credential.service_account_email + name = path_template.expand( + "projects/{project}/serviceAccounts/{service_account}", + project="-", + service_account=service_account_email, + ) + scope = [ + "https://www.googleapis.com/auth/devstorage.read_write", + "https://www.googleapis.com/auth/iam", + ] + response = universe_domain_iam_client.generate_access_token(name=name, scope=scope) + bucket_name = _helpers.unique_name("gcp-systest-ud-sign-blob") + ud_bucket = universe_domain_client.create_bucket( + bucket_name, location=test_universe_location + ) + buckets_to_delete.append(ud_bucket) + + _create_signed_read_url_helper( + universe_domain_client, + ud_bucket, + version="v4", + service_account_email=service_account_email, + access_token=response.access_token, + ) + + def _create_signed_delete_url_helper(client, bucket, version="v2", expiration=None): expiration = _morph_expiration(version, expiration) diff --git a/tests/unit/test_blob.py b/tests/unit/test_blob.py index b0ff4f07b..d805017b9 100644 --- a/tests/unit/test_blob.py +++ b/tests/unit/test_blob.py @@ -487,6 +487,8 @@ def _generate_signed_url_helper( expected_creds = credentials client = self._make_client(_credentials=object()) + expected_universe_domain = client.universe_domain + bucket = _Bucket(client) blob = self._make_one(blob_name, bucket=bucket, encryption_key=encryption_key) @@ -564,6 +566,7 @@ def _generate_signed_url_helper( "query_parameters": query_parameters, "access_token": access_token, "service_account_email": service_account_email, + "universe_domain": expected_universe_domain, } signer.assert_called_once_with(expected_creds, **expected_kwargs) From b245bbf0479463543f90e1605b5522cc23ac9536 Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Sat, 16 Nov 2024 00:40:30 +0000 Subject: [PATCH 3/4] update test credentials --- tests/system/test__signing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system/test__signing.py b/tests/system/test__signing.py index f1b83f771..d20996e38 100644 --- a/tests/system/test__signing.py +++ b/tests/system/test__signing.py @@ -291,11 +291,11 @@ def test_create_signed_read_url_v4_w_access_token_universe_domain( universe_domain_iam_client, universe_domain_client, test_universe_location, - test_universe_domain_credential, + universe_domain_credential, buckets_to_delete, no_mtls, ): - service_account_email = test_universe_domain_credential.service_account_email + service_account_email = universe_domain_credential.service_account_email name = path_template.expand( "projects/{project}/serviceAccounts/{service_account}", project="-", From b8f09c43bbc84c9dfc7db58c028c8895d52a0332 Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Sat, 16 Nov 2024 01:39:33 +0000 Subject: [PATCH 4/4] use ud signing bucket fixture --- tests/system/conftest.py | 19 +++++++++++++++++-- tests/system/test__signing.py | 9 ++------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/tests/system/conftest.py b/tests/system/conftest.py index 7c045fe36..588f66f79 100644 --- a/tests/system/conftest.py +++ b/tests/system/conftest.py @@ -386,6 +386,21 @@ def universe_domain_client( yield ud_storage_client +@pytest.fixture(scope="function") +def universe_domain_bucket(universe_domain_client, test_universe_location): + bucket_name = _helpers.unique_name("gcp-systest-ud") + bucket = universe_domain_client.create_bucket( + bucket_name, location=test_universe_location + ) + + blob = bucket.blob("README.txt") + blob.upload_from_string(_helpers.signing_blob_content) + + yield bucket + + _helpers.delete_bucket(bucket) + + @pytest.fixture(scope="function") def universe_domain_iam_client( test_universe_domain, test_universe_project_id, universe_domain_credential @@ -397,5 +412,5 @@ def universe_domain_iam_client( credentials=universe_domain_credential, client_options=client_options, ) - with contextlib.closing(iam_client): - yield iam_client + + return iam_client diff --git a/tests/system/test__signing.py b/tests/system/test__signing.py index d20996e38..ee7a85fb7 100644 --- a/tests/system/test__signing.py +++ b/tests/system/test__signing.py @@ -292,7 +292,7 @@ def test_create_signed_read_url_v4_w_access_token_universe_domain( universe_domain_client, test_universe_location, universe_domain_credential, - buckets_to_delete, + universe_domain_bucket, no_mtls, ): service_account_email = universe_domain_credential.service_account_email @@ -306,15 +306,10 @@ def test_create_signed_read_url_v4_w_access_token_universe_domain( "https://www.googleapis.com/auth/iam", ] response = universe_domain_iam_client.generate_access_token(name=name, scope=scope) - bucket_name = _helpers.unique_name("gcp-systest-ud-sign-blob") - ud_bucket = universe_domain_client.create_bucket( - bucket_name, location=test_universe_location - ) - buckets_to_delete.append(ud_bucket) _create_signed_read_url_helper( universe_domain_client, - ud_bucket, + universe_domain_bucket, version="v4", service_account_email=service_account_email, access_token=response.access_token,