From d7f983b48de3ed25d8189923429a68e4396709dc Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Thu, 21 Mar 2019 17:20:44 +0000 Subject: [PATCH 1/8] Add globus auth code --- dkist/utils/globus/__init__.py | 3 + dkist/utils/globus/auth.py | 183 +++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 dkist/utils/globus/__init__.py create mode 100644 dkist/utils/globus/auth.py diff --git a/dkist/utils/globus/__init__.py b/dkist/utils/globus/__init__.py new file mode 100644 index 00000000..598a5219 --- /dev/null +++ b/dkist/utils/globus/__init__.py @@ -0,0 +1,3 @@ +""" +Utilities and Helpers for dealing with Globus. +""" diff --git a/dkist/utils/globus/auth.py b/dkist/utils/globus/auth.py new file mode 100644 index 00000000..27235ef7 --- /dev/null +++ b/dkist/utils/globus/auth.py @@ -0,0 +1,183 @@ +""" +Globus Auth Helpers. +""" +# A lot of this code is copied from the Globus Auth Example repo: +# https://github.com/globus/native-app-examples + +import json +import queue +import stat +import threading +import webbrowser +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path +from urllib.parse import parse_qs, urlparse + +import appdirs +import globus_sdk +import globus_sdk.auth.token_response + +CLIENT_ID = 'dd2d62af-0b44-4e2e-9454-1092c94b46b3' +SCOPES = ('urn:globus:auth:scope:transfer.api.globus.org:all',) + + +__all__ = ['get_refresh_token_authorizer'] + + +class RedirectHTTPServer(HTTPServer): + def __init__(self, listen, handler_class): + super().__init__(listen, handler_class) + + self._auth_code_queue = queue.Queue() + + def return_code(self, code): + self._auth_code_queue.put_nowait(code) + + def wait_for_code(self): + return self._auth_code_queue.get(block=True) + + +class RedirectHandler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header('Content-type', 'text/html') + self.end_headers() + self.wfile.write(b'You\'re all set, you can close this window!') + + code = parse_qs(urlparse(self.path).query).get('code', [''])[0] + self.server.return_code(code) + + def log_message(self, format, *args): + return + + +def start_local_server(listen=('localhost', 0)): + """ + Start a server which will listen for the OAuth2 callback. + + Parameters + ---------- + listen: `tuple`, optional + ``(address, port)`` tuple, defaults to localhost and port 0, which + leads to the system choosing a free port. + """ + server = RedirectHTTPServer(listen, RedirectHandler) + thread = threading.Thread(target=server.serve_forever) + thread.daemon = True + thread.start() + + return server + + +def get_cache_file_path(): + """ + Use appdirs to get the cache path for the user and add the filename. + """ + cache_dir = Path(appdirs.user_cache_dir("dkist")) + return cache_dir / "globus_auth_cache.json" + + +def get_cache_contents(): + """ + Read the cache file, return an empty dict if not found or invalid. + """ + cache_file = get_cache_file_path() + if not cache_file.exists(): + return {} + else: + try: + with open(cache_file) as fd: + return json.load(fd) + except (IOError, json.JSONDecodeError): + return {} + + +def save_auth_cache(auth_cache): + """ + Write the auth cache to the cache file. + + Parameters + ---------- + auth_cache: `dict` or `~globus_sdk.auth.token_response.OAuthTokenResponse` + The auth cache to save. + + """ + if isinstance(auth_cache, globus_sdk.auth.token_response.OAuthTokenResponse): + auth_cache = auth_cache.by_resource_server + + cache_file = get_cache_file_path() + + # Ensure the cache dir exists. + cache_dir = cache_file.parent + if not cache_dir.exists(): + cache_dir.mkdir() + + # Write the token to the cache file. + with open(cache_file, "w") as fd: + json.dump(auth_cache, fd) + + # Ensure the token file has minimal permissions. + cache_file.chmod(stat.S_IRUSR | stat.S_IWUSR) + + +def do_native_app_authentication(client_id, requested_scopes=None): + """ + Does a Native App authentication flow and returns a + dict of tokens keyed by service name. + """ + server = start_local_server() + redirect_uri = "http://{a[0]}:{a[1]}".format(a=server.server_address) + + client = globus_sdk.NativeAppAuthClient(client_id=client_id) + client.oauth2_start_flow(requested_scopes=SCOPES, + redirect_uri=redirect_uri, + refresh_tokens=True) + url = client.oauth2_get_authorize_url() + + print("Opening Globus login in your webbrowser...") + webbrowser.open(url, new=1) + + auth_code = server.wait_for_code() + token_response = client.oauth2_exchange_code_for_tokens(auth_code) + + server.shutdown() + + # return a set of tokens, organized by resource server name + return token_response.by_resource_server + + +def get_refresh_token_authorizer(force_reauth=False): + """ + Perform OAuth2 Authentication to Globus. + + Parameters + ---------- + force_reauth: `bool`, optional + If `True` ignore any cached credentials and reauth with Globus. This is + useful if the cache is corrupted or the refresh token has expired. + + Returns + ------- + `globus_sdk.RefreshTokenAuthorizer` + + """ + if not force_reauth: + tokens = get_cache_contents() + else: + tokens = None + + if not tokens: + tokens = do_native_app_authentication(CLIENT_ID, SCOPES) + save_auth_cache(tokens) + + transfer_tokens = tokens['transfer.api.globus.org'] + + auth_client = globus_sdk.NativeAppAuthClient(client_id=CLIENT_ID) + authorizer = globus_sdk.RefreshTokenAuthorizer( + transfer_tokens['refresh_token'], + auth_client, + access_token=transfer_tokens['access_token'], + expires_at=transfer_tokens['expires_at_seconds'], + on_refresh=save_auth_cache) + + return authorizer From f2889e5727a6ce2a43056dc75c841d6f85172603 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Thu, 21 Mar 2019 17:34:03 +0000 Subject: [PATCH 2/8] Comment --- dkist/utils/globus/auth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dkist/utils/globus/auth.py b/dkist/utils/globus/auth.py index 27235ef7..4ba515e3 100644 --- a/dkist/utils/globus/auth.py +++ b/dkist/utils/globus/auth.py @@ -137,6 +137,7 @@ def do_native_app_authentication(client_id, requested_scopes=None): print("Opening Globus login in your webbrowser...") webbrowser.open(url, new=1) + # TODO: Come up with some way of interrupting this or timing it out. auth_code = server.wait_for_code() token_response = client.oauth2_exchange_code_for_tokens(auth_code) From 7b089e9fa4ad6c231781dba6dca26905dff8a376 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Thu, 21 Mar 2019 17:41:24 +0000 Subject: [PATCH 3/8] Add proper handling of KeyboardInterrupt during auth --- dkist/utils/globus/auth.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/dkist/utils/globus/auth.py b/dkist/utils/globus/auth.py index 4ba515e3..69d215e9 100644 --- a/dkist/utils/globus/auth.py +++ b/dkist/utils/globus/auth.py @@ -24,6 +24,12 @@ __all__ = ['get_refresh_token_authorizer'] +class AuthenticationError(Exception): + """ + An error to be raised if authentication fails. + """ + + class RedirectHTTPServer(HTTPServer): def __init__(self, listen, handler_class): super().__init__(listen, handler_class) @@ -134,14 +140,17 @@ def do_native_app_authentication(client_id, requested_scopes=None): refresh_tokens=True) url = client.oauth2_get_authorize_url() - print("Opening Globus login in your webbrowser...") webbrowser.open(url, new=1) - # TODO: Come up with some way of interrupting this or timing it out. - auth_code = server.wait_for_code() - token_response = client.oauth2_exchange_code_for_tokens(auth_code) + print("Waiting for completion of Globus Authentication in your webbrowser...") + try: + auth_code = server.wait_for_code() + except KeyboardInterrupt: + raise AuthenticationError("Failed to authenticate with Globus.") + finally: + server.shutdown() - server.shutdown() + token_response = client.oauth2_exchange_code_for_tokens(auth_code) # return a set of tokens, organized by resource server name return token_response.by_resource_server @@ -162,11 +171,9 @@ def get_refresh_token_authorizer(force_reauth=False): `globus_sdk.RefreshTokenAuthorizer` """ + tokens = None if not force_reauth: tokens = get_cache_contents() - else: - tokens = None - if not tokens: tokens = do_native_app_authentication(CLIENT_ID, SCOPES) save_auth_cache(tokens) From 2eb73678a67a00b7f057a5abff73ac3fe165847c Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 25 Mar 2019 10:50:28 +0000 Subject: [PATCH 4/8] Add globus sdk to deps --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 6fd380d7..ed44c5db 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,6 +24,7 @@ install_requires = numpy scipy matplotlib + globus-sdk # Git deps astropy asdf From 967dec282a5861fb06928e90a520bdf210e199fd Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 25 Mar 2019 10:51:01 +0000 Subject: [PATCH 5/8] Add some tests --- dkist/utils/globus/__init__.py | 1 + dkist/utils/globus/auth.py | 4 ++-- dkist/utils/globus/tests/test_auth.py | 10 ++++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 dkist/utils/globus/tests/test_auth.py diff --git a/dkist/utils/globus/__init__.py b/dkist/utils/globus/__init__.py index 598a5219..8057b5f7 100644 --- a/dkist/utils/globus/__init__.py +++ b/dkist/utils/globus/__init__.py @@ -1,3 +1,4 @@ """ Utilities and Helpers for dealing with Globus. """ +from .auth import get_refresh_token_authorizer diff --git a/dkist/utils/globus/auth.py b/dkist/utils/globus/auth.py index 69d215e9..ea4afaa0 100644 --- a/dkist/utils/globus/auth.py +++ b/dkist/utils/globus/auth.py @@ -5,12 +5,12 @@ # https://github.com/globus/native-app-examples import json -import queue import stat +import queue import threading import webbrowser -from http.server import BaseHTTPRequestHandler, HTTPServer from pathlib import Path +from http.server import HTTPServer, BaseHTTPRequestHandler from urllib.parse import parse_qs, urlparse import appdirs diff --git a/dkist/utils/globus/tests/test_auth.py b/dkist/utils/globus/tests/test_auth.py new file mode 100644 index 00000000..f110e3ac --- /dev/null +++ b/dkist/utils/globus/tests/test_auth.py @@ -0,0 +1,10 @@ +from unittest import mock + +import pytest +import requests + +from dkist.utils.globus.auth import start_local_server + + +def test_http_server(): + server = start_local_server() From 15bad4cfad5077b4a975ee876788f7577701f6be Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 25 Mar 2019 14:20:07 +0000 Subject: [PATCH 6/8] Add tests for globus auth tools --- dkist/utils/globus/auth.py | 2 +- dkist/utils/globus/tests/test_auth.py | 95 ++++++++++++++++++++++++++- 2 files changed, 94 insertions(+), 3 deletions(-) diff --git a/dkist/utils/globus/auth.py b/dkist/utils/globus/auth.py index ea4afaa0..104af460 100644 --- a/dkist/utils/globus/auth.py +++ b/dkist/utils/globus/auth.py @@ -126,7 +126,7 @@ def save_auth_cache(auth_cache): cache_file.chmod(stat.S_IRUSR | stat.S_IWUSR) -def do_native_app_authentication(client_id, requested_scopes=None): +def do_native_app_authentication(client_id, requested_scopes=None): # pragma: no cover """ Does a Native App authentication flow and returns a dict of tokens keyed by service name. diff --git a/dkist/utils/globus/tests/test_auth.py b/dkist/utils/globus/tests/test_auth.py index f110e3ac..b3a48d4d 100644 --- a/dkist/utils/globus/tests/test_auth.py +++ b/dkist/utils/globus/tests/test_auth.py @@ -1,10 +1,101 @@ +import json +import stat +import pathlib from unittest import mock -import pytest import requests +import globus_sdk -from dkist.utils.globus.auth import start_local_server +from dkist.utils.globus.auth import (get_cache_contents, get_cache_file_path, save_auth_cache, + start_local_server, get_refresh_token_authorizer) def test_http_server(): server = start_local_server() + redirect_uri = "http://{a[0]}:{a[1]}".format(a=server.server_address) + inp_code = "wibble" + + requests.get(redirect_uri + f"?code={inp_code}") + + code = server.wait_for_code() + + assert code == inp_code + + +@mock.patch("appdirs.user_cache_dir", return_value="/tmp/test/") +def test_get_cache_file_path(mock_appdirs): + path = get_cache_file_path() + assert isinstance(path, pathlib.Path) + + assert path.parent == pathlib.Path("/tmp/test") + assert path.name == "globus_auth_cache.json" + + +def test_get_no_cache(tmpdir): + with mock.patch("appdirs.user_cache_dir", return_value=str(tmpdir)): + # Test file not exists + cache = get_cache_contents() + assert isinstance(cache, dict) + assert not cache + + +def test_get_cache(tmpdir): + with mock.patch("appdirs.user_cache_dir", return_value=str(tmpdir)): + with open(tmpdir / "globus_auth_cache.json", "w") as fd: + json.dump({"hello": "world"}, fd) + + cache = get_cache_contents() + assert isinstance(cache, dict) + assert len(cache) == 1 + assert cache == {"hello": "world"} + + +def test_get_cache_not_json(tmpdir): + with mock.patch("appdirs.user_cache_dir", return_value=str(tmpdir)): + with open(tmpdir / "globus_auth_cache.json", "w") as fd: + fd.write("aslkjdasdjjdlsajdjklasjdj, akldjaskldjasd, lkjasdkljasldkjas") + + cache = get_cache_contents() + assert isinstance(cache, dict) + assert not cache + + +def test_save_auth_cache(tmpdir): + filename = tmpdir / "globus_auth_cache.json" + assert not filename.exists() # Sanity check + with mock.patch("appdirs.user_cache_dir", return_value=str(tmpdir)): + save_auth_cache({"hello": "world"}) + + assert filename.exists() + statinfo = filename.stat() + + # Test that the user can read and write + assert bool(statinfo.mode & stat.S_IRUSR) + assert bool(statinfo.mode & stat.S_IWUSR) + # Test that neither "Group" or "Other" have read permissions + assert not bool(statinfo.mode & stat.S_IRGRP) + assert not bool(statinfo.mode & stat.S_IROTH) + + +def test_get_refresh_token_authorizer(): + # An example cache without real tokens + cache = { + "transfer.api.globus.org": { + "scope": "urn:globus:auth:scope:transfer.api.globus.org:all", + "access_token": "buscVeATmhfB0v1tzu8VmTfFRB1nwlF8bn1R9rQTI3Q", + "refresh_token": "YSbLZowAHfmhxehUqeOF3lFvoC0FlTT11QGupfWAOX4", + "token_type": "Bearer", + "expires_at_seconds": 1553362861, + "resource_server": "transfer.api.globus.org" + } + } + + with mock.patch("dkist.utils.globus.auth.get_cache_contents", return_value=cache): + auth = get_refresh_token_authorizer() + assert isinstance(auth, globus_sdk.RefreshTokenAuthorizer) + assert auth.access_token == cache["transfer.api.globus.org"]["access_token"] + + with mock.patch("dkist.utils.globus.auth.do_native_app_authentication", return_value=cache): + auth = get_refresh_token_authorizer(force_reauth=True) + assert isinstance(auth, globus_sdk.RefreshTokenAuthorizer) + assert auth.access_token == cache["transfer.api.globus.org"]["access_token"] From 89061330d4b1843fc7a1e9b25504ebe7dd9ac850 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 25 Mar 2019 16:11:14 +0000 Subject: [PATCH 7/8] cleanup CI config and add appdirs as a dep --- setup.cfg | 1 + tox.ini | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/setup.cfg b/setup.cfg index ed44c5db..e937ec2b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,6 +25,7 @@ install_requires = scipy matplotlib globus-sdk + appdirs # Git deps astropy asdf diff --git a/tox.ini b/tox.ini index e27f5d83..d5112661 100644 --- a/tox.ini +++ b/tox.ini @@ -6,15 +6,10 @@ setenv = MPLBACKEND = agg passenv = CC deps = - numpy - scipy - matplotlib - dask[array] coverage pytest-astropy pytest-cov pytest-mpl - importlib_resources;python_version<"3.7" git+https://github.com/astropy/astropy git+https://github.com/spacetelescope/asdf git+https://github.com/Cadair/gwcs@dkist From c63769f46ae3bca78e19f0c582a9fd57d9d6fcd2 Mon Sep 17 00:00:00 2001 From: Stuart Mumford Date: Mon, 25 Mar 2019 16:38:10 +0000 Subject: [PATCH 8/8] Add changelog --- changelog/46.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/46.feature.rst diff --git a/changelog/46.feature.rst b/changelog/46.feature.rst new file mode 100644 index 00000000..061c8811 --- /dev/null +++ b/changelog/46.feature.rst @@ -0,0 +1 @@ +Add utilities for doing OAuth with Globus.