From 5e0464f1a92fd48b2b02408811fa4fb31286806f Mon Sep 17 00:00:00 2001 From: Kevin Guillaumond Date: Wed, 19 Jun 2019 15:52:00 -0700 Subject: [PATCH 1/3] refactor(auth): update auth classes --- .pre-commit-config.yaml | 1 - gcloud/rest/auth/__init__.py | 5 +- gcloud/rest/auth/iam.py | 149 ++++++++++++++++++++ gcloud/rest/auth/token.py | 240 +++++++++++++++++++++++---------- gcloud/rest/auth/utils.py | 25 ++++ gcloud/rest/kms/client.py | 3 +- gcloud/rest/storage/bucket.py | 4 +- gcloud/rest/taskqueue/queue.py | 2 +- requirements.txt | 2 + tests/unit/auth/iam_test.py | 5 + tests/unit/auth/utils_test.py | 19 +++ 11 files changed, 381 insertions(+), 74 deletions(-) create mode 100644 gcloud/rest/auth/iam.py create mode 100644 gcloud/rest/auth/utils.py create mode 100644 tests/unit/auth/iam_test.py create mode 100644 tests/unit/auth/utils_test.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 64d8be5..eff43b9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -74,7 +74,6 @@ repos: hooks: - id: python-no-eval - id: python-no-log-warn - - id: python-use-type-annotations - id: rst-backticks - repo: https://github.com/Lucas-C/pre-commit-hooks-markup rev: v1.0.0 diff --git a/gcloud/rest/auth/__init__.py b/gcloud/rest/auth/__init__.py index 743ca65..3591306 100644 --- a/gcloud/rest/auth/__init__.py +++ b/gcloud/rest/auth/__init__.py @@ -1,7 +1,10 @@ from pkg_resources import get_distribution __version__ = get_distribution('gcloud-rest').version +from gcloud.rest.auth.iam import IamClient from gcloud.rest.auth.token import Token +from gcloud.rest.auth.utils import decode +from gcloud.rest.auth.utils import encode -__all__ = ['__version__', 'Token'] +__all__ = ['__version__', 'IamClient', 'Token', 'decode', 'encode'] diff --git a/gcloud/rest/auth/iam.py b/gcloud/rest/auth/iam.py new file mode 100644 index 0000000..10a9c80 --- /dev/null +++ b/gcloud/rest/auth/iam.py @@ -0,0 +1,149 @@ +import json +import threading +from typing import Dict +from typing import List +from typing import Optional +from typing import Union # pylint: disable=unused-import + +import requests + +from .token import Token +from .token import Type +from .utils import encode + + +API_ROOT_IAM = 'https://iam.googleapis.com/v1' +API_ROOT_IAM_CREDENTIALS = 'https://iamcredentials.googleapis.com/v1' +SCOPES = ['https://www.googleapis.com/auth/iam'] + + +class IamClient: + def __init__(self, + service_file=None, # type: Optional[str] + session=None, # type: requests.Session + google_api_lock=None, # type: threading.RLock + token=None # type: Token + ): + # type: (...) -> None + self.session = session + self.google_api_lock = google_api_lock or threading.RLock() + self.token = token or Token(service_file=service_file, + session=session, scopes=SCOPES) + + if self.token.token_type != Type.SERVICE_ACCOUNT: + raise TypeError('IAM Credentials Client is only valid for use' + ' with Service Accounts') + + def headers(self): + # type: () -> Dict[str, str] + token = self.token.get() + return { + 'Authorization': 'Bearer {}'.format(token), + } + + @property + def service_account_email(self): + # type: () -> Optional[str] + return self.token.service_data.get('client_email') + + # https://cloud.google.com/iam/reference/rest/v1/projects.serviceAccounts.keys/get + # pylint: disable=too-many-arguments + def get_public_key(self, + key_id=None, # type: Optional[str] + key=None, # type: Optional[str] + service_account_email=None, # type: Optional[str] + project=None, # type: Optional[str] + session=None, # type: requests.Session + timeout=10 # type: int + ): + # type: (...) -> Dict[str, str] + service_account_email = (service_account_email + or self.service_account_email) + project = project or self.token.get_project() + + if not key_id and not key: + raise ValueError('get_public_key must have either key_id or key') + + if not key: + key = 'projects/{}/serviceAccounts/{}/keys/{}' \ + .format(project, service_account_email, key_id) + + url = '{}/{}?publicKeyType=TYPE_X509_PEM_FILE'.format( + API_ROOT_IAM, key) + headers = self.headers() + + if not self.session: + self.session = requests.Session() + + session = session or self.session + with self.google_api_lock: + resp = session.get(url, headers=headers, timeout=timeout) + resp.raise_for_status() + return resp.json() + + # https://cloud.google.com/iam/reference/rest/v1/projects.serviceAccounts.keys/list + def list_public_keys(self, + service_account_email=None, # type: Optional[str] + project=None, # type: Optional[str] + session=None, # type: requests.Session + timeout=10 # type: int + ): + # type: (...) -> List[Dict[str, str]] + service_account_email = (service_account_email + or self.service_account_email) + project = project or self.token.get_project() + + url = ('{}/projects/{}/serviceAccounts/' + '{}/keys').format(API_ROOT_IAM, project, service_account_email) + + headers = self.headers() + + if not self.session: + self.session = requests.Session() + + session = session or self.session + with self.google_api_lock: + resp = session.get(url, headers=headers, timeout=timeout) + resp.raise_for_status() + return resp.json().get('keys', []) + + # https://cloud.google.com/iam/credentials/reference/rest/v1/projects.serviceAccounts/signBlob + # pylint: disable=too-many-arguments + def sign_blob(self, + payload, # type: Optional[Union[str, bytes]] + service_account_email=None, # type: Optional[str] + delegates=None, # type: Optional[list] + session=None, # type: requests.Session + timeout=10 # type: int + ): + # type: (...) -> Dict[str, str] + service_account_email = ( + service_account_email or self.service_account_email) + if not service_account_email: + raise TypeError('sign_blob must have a valid ' + 'service_account_email') + + resource_name = 'projects/-/serviceAccounts/{}'.format( + service_account_email) + url = '{}/{}:signBlob'.format(API_ROOT_IAM_CREDENTIALS, resource_name) + + json_str = json.dumps({ + 'delegates': delegates or [resource_name], + 'payload': encode(payload).decode('utf-8'), + }) + + headers = self.headers() + headers.update({ + 'Content-Length': str(len(json_str)), + 'Content-Type': 'application/json', + }) + + if not self.session: + self.session = requests.Session() + + session = session or self.session + with self.google_api_lock: + resp = session.post(url, data=json_str, + headers=headers, timeout=timeout) + resp.raise_for_status() + return resp.json() diff --git a/gcloud/rest/auth/token.py b/gcloud/rest/auth/token.py index cbe8514..aea6852 100644 --- a/gcloud/rest/auth/token.py +++ b/gcloud/rest/auth/token.py @@ -1,95 +1,199 @@ +"""Google Cloud auth via service account file""" import datetime +import enum import json -import logging import os import threading import time +from typing import Any +from typing import Dict +from typing import List # pylint: disable=unused-import +from typing import Optional + try: from urllib.parse import urlencode + from urllib.parse import quote_plus except ImportError: from urllib import urlencode + from urllib import quote_plus -# N.B. the cryptography library is required when calling jwt.encrypt() with -# algorithm='RS256'. It does not need to be imported here, but this allows us -# to throw this error at load time rather than lazily during normal operations, -# where plumbing this error through will require several changes to otherwise- -# good error handling. -import cryptography # pylint: disable=unused-import +import backoff import jwt import requests -TOKEN_URI = 'https://accounts.google.com/o/oauth2/token' -TIMEOUT = 60 - -log = logging.getLogger(__name__) +GCE_METADATA_BASE = 'http://metadata.google.internal/computeMetadata/v1' +GCE_METADATA_HEADERS = {'metadata-flavor': 'Google'} +GCE_ENDPOINT_PROJECT = '{}/project/project-id'.format(GCE_METADATA_BASE) +GCE_ENDPOINT_TOKEN = \ + '{}/instance/service-accounts/default/token?recursive=true'\ + .format(GCE_METADATA_BASE) +GCLOUD_TOKEN_DURATION = 3600 +REFRESH_HEADERS = {'Content-Type': 'application/x-www-form-urlencoded'} + + +class Type(enum.Enum): + AUTHORIZED_USER = 'authorized_user' + GCE_METADATA = 'gce_metadata' + SERVICE_ACCOUNT = 'service_account' + + +def get_service_data(service): + # type: (Optional[str]) -> Dict[str, Any] + service = service or os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') + if not service: + cloudsdk_config = os.environ.get('CLOUDSDK_CONFIG') + sdkpath = (cloudsdk_config + or os.path.join(os.path.expanduser('~'), '.config', + 'gcloud')) + service = os.path.join(sdkpath, 'application_default_credentials.json') + set_explicitly = bool(cloudsdk_config) + else: + set_explicitly = True + + try: + with open(service, 'r') as f: + data = json.loads(f.read()) + return data + except FileNotFoundError: + # only warn users if they have explicitly set the service_file path + if set_explicitly: + raise + return {} + except Exception: # pylint: disable=broad-except + return {} class Token(object): - def __init__(self, creds=None, google_api_lock=None, scopes=None, - timeout=TIMEOUT): - self.creds = creds or os.getenv('GOOGLE_APPLICATION_CREDENTIALS') - if not self.creds: - raise Exception('could not load service credentials') + # pylint: disable=too-many-instance-attributes + def __init__(self, + service_file=None, # type: Optional[str] + session=None, # type: requests.Session + google_api_lock=None, # type: threading.RLock + scopes=None # type: List[str] + ): + # type: (...) -> None + self.service_data = get_service_data(service_file) + if self.service_data: + self.token_type = Type(self.service_data['type']) + self.token_uri = self.service_data.get( + 'token_uri', 'https://oauth2.googleapis.com/token') + else: + # At this point, all we can do is assume we're running somewhere + # with default credentials, eg. GCE. + self.token_type = Type.GCE_METADATA + self.token_uri = GCE_ENDPOINT_TOKEN self.google_api_lock = google_api_lock or threading.RLock() - self.scopes = scopes or [] - self.timeout = timeout + self.session = session + self.scopes = ' '.join(scopes or []) + if self.token_type == Type.SERVICE_ACCOUNT and not self.scopes: + raise Exception('scopes must be provided when token type is ' + 'service account') + + self.access_token = None + self.access_token_duration = 0 + self.access_token_acquired_at = datetime.datetime(1970, 1, 1) + + self.acquiring = None + + def get_project(self): + # type: () -> Optional[str] + project = (os.environ.get('GOOGLE_CLOUD_PROJECT') + or os.environ.get('GCLOUD_PROJECT') + or os.environ.get('APPLICATION_ID')) + + if self.token_type == Type.GCE_METADATA: + self.ensure_token() + with self.google_api_lock: + resp = self.session.get(GCE_ENDPOINT_PROJECT, timeout=10, + headers=GCE_METADATA_HEADERS) + resp.raise_for_status() + project = project or resp.text + elif self.token_type == Type.SERVICE_ACCOUNT: + project = project or self.service_data.get('project_id') + + return project + + def get(self): + # type: () -> str + self.ensure_token() + return self.access_token + + def ensure_token(self): + # type: () -> None + if not self.access_token: + self.acquire_access_token() + return + + now = datetime.datetime.utcnow() + delta = (now - self.access_token_acquired_at).total_seconds() + if delta <= self.access_token_duration / 2: + return - self.age = datetime.datetime.now() - self.expiry = 60 - self.value = None + self.acquire_access_token() - def __str__(self): - self.ensure() - return str(self.value) + def _refresh_authorized_user(self, timeout): + # type: (int) -> requests.Response + payload = urlencode({ + 'grant_type': 'refresh_token', + 'client_id': self.service_data['client_id'], + 'client_secret': self.service_data['client_secret'], + 'refresh_token': self.service_data['refresh_token'], + }, quote_via=quote_plus) + with self.google_api_lock: + return self.session.post(self.token_uri, data=payload, + headers=REFRESH_HEADERS, timeout=timeout) - def assertion(self): - with open(self.creds, 'r') as f: - credentials = json.loads(f.read()) + def _refresh_gce_metadata(self, timeout): + # type: (int) -> requests.Response + with self.google_api_lock: + return self.session.get(self.token_uri, + headers=GCE_METADATA_HEADERS, + timeout=timeout) - # N.B. the below exists to avoid using this private method: - # return ServiceAccountCredentials._generate_assertion() + def _refresh_service_account(self, timeout): + # type: (int) -> requests.Response now = int(time.time()) - payload = { - 'aud': TOKEN_URI, - 'exp': now + 3600, + assertion_payload = { + 'aud': self.token_uri, + 'exp': now + GCLOUD_TOKEN_DURATION, 'iat': now, - 'iss': credentials['client_email'], - 'scope': ' '.join(self.scopes), + 'iss': self.service_data['client_email'], + 'scope': self.scopes, } - return jwt.encode(payload, credentials['private_key'], - algorithm='RS256') - - def acquire(self): - headers = {'Content-Type': 'application/x-www-form-urlencoded'} - body = urlencode(( - ('grant_type', 'urn:ietf:params:oauth:grant-type:jwt-bearer'), - ('assertion', self.assertion()), - )) - + # N.B. algorithm='RS256' requires an extra 240MB in dependencies... + assertion = jwt.encode(assertion_payload, + self.service_data['private_key'], + algorithm='RS256') + payload = urlencode({ + 'assertion': assertion, + 'grant_type': 'urn:ietf:params:oauth:grant-type:jwt-bearer', + }, quote_via=quote_plus) with self.google_api_lock: - response = requests.post(TOKEN_URI, data=body, headers=headers, - timeout=self.timeout) - - content = response.json() - if 'error' in content: - raise Exception('{}'.format(content)) - - self.age = datetime.datetime.now() - self.expiry = int(content['expires_in']) - self.value = content['access_token'] - - def ensure(self): - if not self.value: - log.debug('acquiring initial token') - self.acquire() - return - - now = datetime.datetime.now() - delta = (now - self.age).total_seconds() - - if delta > self.expiry / 2: - log.debug('requiring token with expiry %d of %d / 2', delta, - self.expiry) - self.acquire() + return self.session.post(self.token_uri, data=payload, + headers=REFRESH_HEADERS, timeout=timeout) + + @backoff.on_exception(backoff.expo, Exception, max_tries=5) # type: ignore + def acquire_access_token(self, timeout=10): + # type: (int) -> None + if not self.session: + self.session = requests.Session() + + if self.token_type == Type.AUTHORIZED_USER: + resp = self._refresh_authorized_user(timeout=timeout) + elif self.token_type == Type.GCE_METADATA: + resp = self._refresh_gce_metadata(timeout=timeout) + elif self.token_type == Type.SERVICE_ACCOUNT: + resp = self._refresh_service_account(timeout=timeout) + else: + raise Exception( + 'unsupported token type {}'.format(self.token_type)) + + resp.raise_for_status() + content = resp.json() + + self.access_token = str(content['access_token']) + self.access_token_duration = int(content['expires_in']) + self.access_token_acquired_at = datetime.datetime.utcnow() + self.acquiring = None diff --git a/gcloud/rest/auth/utils.py b/gcloud/rest/auth/utils.py new file mode 100644 index 0000000..6ffe5ba --- /dev/null +++ b/gcloud/rest/auth/utils.py @@ -0,0 +1,25 @@ +import base64 +from typing import Union + + +def decode(payload): + # type: (str) -> bytes + """ + Modified Base64 for URL variants exist, where the + and / characters of + standard Base64 are respectively replaced by - and _. + See https://en.wikipedia.org/wiki/Base64#URL_applications + """ + return base64.b64decode(payload, altchars=b'-_') + + +def encode(payload): + # type: (Union[bytes, str]) -> bytes + """ + Modified Base64 for URL variants exist, where the + and / characters of + standard Base64 are respectively replaced by - and _. + See https://en.wikipedia.org/wiki/Base64#URL_applications + """ + if isinstance(payload, str): + payload = payload.encode('utf-8') + + return base64.b64encode(payload, altchars=b'-_') diff --git a/gcloud/rest/kms/client.py b/gcloud/rest/kms/client.py index 0bdc1ed..e173db7 100644 --- a/gcloud/rest/kms/client.py +++ b/gcloud/rest/kms/client.py @@ -1,3 +1,4 @@ +# pylint: disable=duplicate-code import threading import requests @@ -22,7 +23,7 @@ def __init__(self, project, keyring, keyname, creds=None, self.google_api_lock = google_api_lock or threading.RLock() - self.access_token = Token(creds=creds, + self.access_token = Token(service_file=creds, google_api_lock=self.google_api_lock, scopes=SCOPES) diff --git a/gcloud/rest/storage/bucket.py b/gcloud/rest/storage/bucket.py index a857645..59e5dff 100644 --- a/gcloud/rest/storage/bucket.py +++ b/gcloud/rest/storage/bucket.py @@ -1,3 +1,4 @@ +# pylint: disable=duplicate-code import logging import threading @@ -22,8 +23,7 @@ def __init__(self, project, bucket, creds=None, google_api_lock=None): self.bucket = bucket self.google_api_lock = google_api_lock or threading.RLock() - - self.access_token = Token(creds=creds, + self.access_token = Token(service_file=creds, google_api_lock=self.google_api_lock, scopes=SCOPES) diff --git a/gcloud/rest/taskqueue/queue.py b/gcloud/rest/taskqueue/queue.py index e5f5893..e4721d4 100644 --- a/gcloud/rest/taskqueue/queue.py +++ b/gcloud/rest/taskqueue/queue.py @@ -21,7 +21,7 @@ def __init__(self, project, taskqueue, creds=None, google_api_lock=None, self.google_api_lock = google_api_lock or threading.RLock() - self.access_token = Token(creds=creds, + self.access_token = Token(service_file=creds, google_api_lock=self.google_api_lock, scopes=SCOPES) diff --git a/requirements.txt b/requirements.txt index 0cc917b..620363a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ +backoff >= 1.0.0, < 2.0.0 cryptography >= 2.0.0, < 3.0.0 pyjwt >= 1.5.3, < 2.0.0 requests[security] >= 2.0.0, < 3.0.0 +typing >= 3.0.0, < 4.0.0 diff --git a/tests/unit/auth/iam_test.py b/tests/unit/auth/iam_test.py new file mode 100644 index 0000000..f5aef16 --- /dev/null +++ b/tests/unit/auth/iam_test.py @@ -0,0 +1,5 @@ +import gcloud.rest.auth.iam as iam # pylint: disable=unused-import + + +def test_importable(): + assert True diff --git a/tests/unit/auth/utils_test.py b/tests/unit/auth/utils_test.py new file mode 100644 index 0000000..1515520 --- /dev/null +++ b/tests/unit/auth/utils_test.py @@ -0,0 +1,19 @@ +import pickle + +import pytest + +import gcloud.rest.auth.utils as utils + + +@pytest.mark.parametrize('str_or_bytes', ['Hello Test', + 'UTF-8 Bytes'.encode('utf-8'), + pickle.dumps([])]) +def test_encode_decode(str_or_bytes): + encoded = utils.encode(str_or_bytes) + expected = str_or_bytes + if isinstance(expected, str): + try: + expected = str_or_bytes.encode('utf-8') + except UnicodeDecodeError: + pass + assert expected == utils.decode(encoded) From f0bd1475d16918962e22d4d6931a3362c71ebcfb Mon Sep 17 00:00:00 2001 From: Gunjan Korlekar Date: Thu, 27 Jun 2019 15:30:01 -0700 Subject: [PATCH 2/3] address review comments --- .pre-commit-config.py27.yaml | 10 ++---- .pre-commit-config.yaml | 4 ++- README.rst | 2 +- gcloud/rest/auth/iam.py | 16 +++++----- gcloud/rest/auth/token.py | 57 +++++++++++++++++++++++++++++------ gcloud/rest/auth/utils.py | 2 +- gcloud/rest/kms/client.py | 1 - gcloud/rest/storage/bucket.py | 1 - 8 files changed, 62 insertions(+), 31 deletions(-) diff --git a/.pre-commit-config.py27.yaml b/.pre-commit-config.py27.yaml index 292566d..0149127 100644 --- a/.pre-commit-config.py27.yaml +++ b/.pre-commit-config.py27.yaml @@ -32,22 +32,19 @@ repos: args: - --max-line-length=79 - --ignore-imports=yes + - -d bad-continuation - -d fixme - -d import-error - -d invalid-name - -d locally-disabled - -d missing-docstring - -d too-few-public-methods + - -d too-many-arguments - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.1.6 hooks: - id: remove-crlf - id: remove-tabs -- repo: https://github.com/asottile/reorder_python_imports - rev: v1.5.0 - hooks: - - id: reorder-python-imports - args: [--py26-plus] - repo: https://github.com/asottile/yesqa rev: v0.0.11 hooks: @@ -60,5 +57,4 @@ repos: rev: v1.4.0 hooks: - id: python-no-eval - - id: python-no-log-warn - - id: python-use-type-annotations + # - id: python-no-log-warn TODO: fix the hook to not catch 'warnings.warn' and uncomment once done diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eff43b9..75486f7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,12 +32,14 @@ repos: args: - --max-line-length=79 - --ignore-imports=yes + - -d duplicate-code - -d fixme - -d import-error - -d invalid-name - -d locally-disabled - -d missing-docstring - -d too-few-public-methods + - -d too-many-arguments - -d useless-object-inheritance # necessary for Python 2 compatibility - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.1.6 @@ -73,7 +75,7 @@ repos: rev: v1.4.0 hooks: - id: python-no-eval - - id: python-no-log-warn + # - id: python-no-log-warn TODO: fix the hook to not catch 'warnings.warn' and uncomment once done - id: rst-backticks - repo: https://github.com/Lucas-C/pre-commit-hooks-markup rev: v1.0.0 diff --git a/README.rst b/README.rst index 689719a..2e337df 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ RESTful Google Cloud Client Library for Python ============================================== This project is a collection of Google Cloud client libraries for the REST-only -APIs; its *raison d'être* is to implement a simple `CloudTasks API`_ as well as +APIs; its *raison d'etre* is to implement a simple `CloudTasks API`_ as well as a more abstract TaskManager. If you don't need to support Python 2, you probably want to use `gcloud-aio`_, diff --git a/gcloud/rest/auth/iam.py b/gcloud/rest/auth/iam.py index 10a9c80..4392636 100644 --- a/gcloud/rest/auth/iam.py +++ b/gcloud/rest/auth/iam.py @@ -1,8 +1,8 @@ import json import threading -from typing import Dict -from typing import List -from typing import Optional +from typing import Dict # pylint: disable=unused-import +from typing import List # pylint: disable=unused-import +from typing import Optional # pylint: disable=unused-import from typing import Union # pylint: disable=unused-import import requests @@ -17,12 +17,12 @@ SCOPES = ['https://www.googleapis.com/auth/iam'] -class IamClient: +class IamClient(object): def __init__(self, service_file=None, # type: Optional[str] - session=None, # type: requests.Session - google_api_lock=None, # type: threading.RLock - token=None # type: Token + session=None, # type: Optional[requests.Session] + google_api_lock=None, # type: Optional[threading.RLock] + token=None # type: Optional[Token] ): # type: (...) -> None self.session = session @@ -47,7 +47,6 @@ def service_account_email(self): return self.token.service_data.get('client_email') # https://cloud.google.com/iam/reference/rest/v1/projects.serviceAccounts.keys/get - # pylint: disable=too-many-arguments def get_public_key(self, key_id=None, # type: Optional[str] key=None, # type: Optional[str] @@ -108,7 +107,6 @@ def list_public_keys(self, return resp.json().get('keys', []) # https://cloud.google.com/iam/credentials/reference/rest/v1/projects.serviceAccounts/signBlob - # pylint: disable=too-many-arguments def sign_blob(self, payload, # type: Optional[Union[str, bytes]] service_account_email=None, # type: Optional[str] diff --git a/gcloud/rest/auth/token.py b/gcloud/rest/auth/token.py index aea6852..0526934 100644 --- a/gcloud/rest/auth/token.py +++ b/gcloud/rest/auth/token.py @@ -5,10 +5,11 @@ import os import threading import time -from typing import Any -from typing import Dict +import warnings +from typing import Any # pylint: disable=unused-import +from typing import Dict # pylint: disable=unused-import from typing import List # pylint: disable=unused-import -from typing import Optional +from typing import Optional # pylint: disable=unused-import try: from urllib.parse import urlencode @@ -18,6 +19,12 @@ from urllib import quote_plus import backoff +# N.B. the cryptography library is required when calling jwt.encrypt() with +# algorithm='RS256'. It does not need to be imported here, but this allows us +# to throw this error at load time rather than lazily during normal operations, +# where plumbing this error through will require several changes to otherwise- +# good error handling. +import cryptography # pylint: disable=unused-import import jwt import requests @@ -55,7 +62,7 @@ def get_service_data(service): with open(service, 'r') as f: data = json.loads(f.read()) return data - except FileNotFoundError: + except (IOError, OSError): # only warn users if they have explicitly set the service_file path if set_explicitly: raise @@ -67,12 +74,24 @@ def get_service_data(service): class Token(object): # pylint: disable=too-many-instance-attributes def __init__(self, + creds=None, # type: Optional[str] + google_api_lock=None, # type: Optional[threading.RLock] + scopes=None, # type: Optional[List[str]] + timeout=None, # type: Optional[int] service_file=None, # type: Optional[str] - session=None, # type: requests.Session - google_api_lock=None, # type: threading.RLock - scopes=None # type: List[str] + session=None, # type: Optional[requests.Session] ): # type: (...) -> None + if creds: + warnings.warn('creds is now deprecated for Token(),' + 'please use service_file instead', + DeprecationWarning) + service_file = creds + if timeout: + warnings.warn( + 'timeout arg is now deprecated for Token()', + DeprecationWarning) + self.service_data = get_service_data(service_file) if self.service_data: self.token_type = Type(self.service_data['type']) @@ -119,8 +138,19 @@ def get(self): self.ensure_token() return self.access_token + def __str__(self): + # type: () -> str + return str(self.get()) + + def acquire(self): + # type: () -> str + warnings.warn('Token.acquire() is deprecated', + 'please use Token.acquire_access_token()', + DeprecationWarning) + return self.acquire_access_token() + def ensure_token(self): - # type: () -> None + # type: () -> None if not self.access_token: self.acquire_access_token() return @@ -132,6 +162,13 @@ def ensure_token(self): self.acquire_access_token() + def ensure(self): + # type: () -> None + warnings.warn('Token.ensure() is deprecated', + 'please use Token.ensure_token()', + DeprecationWarning) + self.ensure_token() + def _refresh_authorized_user(self, timeout): # type: (int) -> requests.Response payload = urlencode({ @@ -169,12 +206,12 @@ def _refresh_service_account(self, timeout): payload = urlencode({ 'assertion': assertion, 'grant_type': 'urn:ietf:params:oauth:grant-type:jwt-bearer', - }, quote_via=quote_plus) + }) with self.google_api_lock: return self.session.post(self.token_uri, data=payload, headers=REFRESH_HEADERS, timeout=timeout) - @backoff.on_exception(backoff.expo, Exception, max_tries=5) # type: ignore + @backoff.on_exception(backoff.expo, Exception, max_tries=1) # type: ignore def acquire_access_token(self, timeout=10): # type: (int) -> None if not self.session: diff --git a/gcloud/rest/auth/utils.py b/gcloud/rest/auth/utils.py index 6ffe5ba..a34a655 100644 --- a/gcloud/rest/auth/utils.py +++ b/gcloud/rest/auth/utils.py @@ -1,5 +1,5 @@ import base64 -from typing import Union +from typing import Union # pylint: disable=unused-import def decode(payload): diff --git a/gcloud/rest/kms/client.py b/gcloud/rest/kms/client.py index e173db7..30b0d6b 100644 --- a/gcloud/rest/kms/client.py +++ b/gcloud/rest/kms/client.py @@ -1,4 +1,3 @@ -# pylint: disable=duplicate-code import threading import requests diff --git a/gcloud/rest/storage/bucket.py b/gcloud/rest/storage/bucket.py index 59e5dff..28e2dc0 100644 --- a/gcloud/rest/storage/bucket.py +++ b/gcloud/rest/storage/bucket.py @@ -1,4 +1,3 @@ -# pylint: disable=duplicate-code import logging import threading From 1e818b96fbe7f61975bed575e2e058e8e32e892e Mon Sep 17 00:00:00 2001 From: Gunjan Korlekar Date: Mon, 1 Jul 2019 16:01:10 -0700 Subject: [PATCH 3/3] Reverting max retries and removing quote_via arg --- gcloud/rest/auth/token.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/gcloud/rest/auth/token.py b/gcloud/rest/auth/token.py index 0526934..8991861 100644 --- a/gcloud/rest/auth/token.py +++ b/gcloud/rest/auth/token.py @@ -13,10 +13,8 @@ try: from urllib.parse import urlencode - from urllib.parse import quote_plus except ImportError: from urllib import urlencode - from urllib import quote_plus import backoff # N.B. the cryptography library is required when calling jwt.encrypt() with @@ -176,7 +174,7 @@ def _refresh_authorized_user(self, timeout): 'client_id': self.service_data['client_id'], 'client_secret': self.service_data['client_secret'], 'refresh_token': self.service_data['refresh_token'], - }, quote_via=quote_plus) + }) with self.google_api_lock: return self.session.post(self.token_uri, data=payload, headers=REFRESH_HEADERS, timeout=timeout) @@ -211,7 +209,7 @@ def _refresh_service_account(self, timeout): return self.session.post(self.token_uri, data=payload, headers=REFRESH_HEADERS, timeout=timeout) - @backoff.on_exception(backoff.expo, Exception, max_tries=1) # type: ignore + @backoff.on_exception(backoff.expo, Exception, max_tries=5) # type: ignore def acquire_access_token(self, timeout=10): # type: (int) -> None if not self.session: