Add Python 3 compatibility (#40)

* Python 3 compatibility

* fix header issue

* update requirements

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 jobs:
   build:
     docker:
-      - image: circleci/python:2.7-stretch-browsers
+      - image: circleci/python:3.6.5-stretch-browsers
       - image: mysql/mysql-server:5.7
         environment:
           - MYSQL_ROOT_PASSWORD=admin

src/iris_relay/app.py

@@ -12,8 +12,16 @@
 from logging import basicConfig, getLogger
 from importlib import import_module
 import logging
-from urllib import unquote_plus, urlencode, unquote
-import urllib2
+# preserve python 2 and 3 compatibility
+try:
+    from urllib import unquote_plus, urlencode, unquote
+except ImportError:
+    from urllib.parse import unquote_plus, urlencode, unquote
+
+try:
+    import urllib2
+except ImportError:
+    import urllib.request as urllib2
 
 from . import db
 from streql import equals
@@ -56,17 +64,20 @@ def compute_signature(token, uri, post_body, utf=False):
     :returns: The computed signature
     """
     s = uri
+    post_body = post_body if isinstance(post_body, bytes) else post_body.encode('utf8')
     if len(post_body) > 0:
+        p_b_split = post_body.decode().split('&')
         lst = [unquote_plus(kv.replace('=', ''))
-               for kv in sorted(post_body.split('&'))]
+               for kv in sorted(p_b_split)]
         lst.insert(0, s)
         s = ''.join(lst)
 
+    s = s if isinstance(s, bytes) else s.encode('utf8')
+    token = token if isinstance(token, bytes) else token.encode('utf8')
+
     # compute signature and compare signatures
-    if isinstance(s, str):
+    if isinstance(s, bytes):
         mac = hmac.new(token, s, sha1)
-    elif isinstance(s, unicode):
-        mac = hmac.new(token, s.encode("utf-8"), sha1)
     else:
         # Should never happen
         raise TypeError
@@ -305,7 +316,9 @@ def __init__(self, config, iclient):
         self.config = config
         self.iclient = iclient
         self.data_keys = ('msg_id', 'email_address', 'cmd')  # Order here matters; needs to match what is in iris-api
-        self.hmac = hmac.new(self.config['gmail_one_click_url_key'], '', sha512)
+        key = self.config['gmail_one_click_url_key']
+        key = key if isinstance(key, bytes) else key.encode('utf8')
+        self.hmac = hmac.new(key, b'', sha512)
 
     def on_get(self, req, resp):
         token = req.get_param('token', True)
@@ -333,7 +346,9 @@ def on_get(self, req, resp):
 
     def validate_token(self, given_token, data):
         mac = self.hmac.copy()
-        mac.update(' '.join(data[key] for key in self.data_keys))
+        text = ' '.join(data[key] for key in self.data_keys)
+        text = text if isinstance(text, bytes) else text.encode('utf8')
+        mac.update(text)
         return given_token == urlsafe_b64encode(mac.digest())
 
 
@@ -574,9 +589,11 @@ def on_post(self, req, resp):
             try:
                 msg_id = int(payload['callback_id'])
             except KeyError as e:
+                logger.error(e)
                 logger.error('callback_id not found in the json payload.')
                 raise falcon.HTTPBadRequest('Bad Request', 'Callback id not found')
             except ValueError as e:
+                logger.error(e)
                 logger.error('Callback ID not an integer: %s', payload['callback_id'])
                 raise falcon.HTTPBadRequest('Bad Request', 'Callback id must be int')
             data = {'msg_id': msg_id,
@@ -753,6 +770,7 @@ def process_request(self, req, resp):
                     post_body = req.context['body']
                     expected_sigs = [compute_signature(t, ''.join(uri), post_body)
                                      for t in self.twilio_auth_token]
+                    sig = sig if isinstance(sig, bytes) else sig.encode('utf8')
                     if sig not in expected_sigs:
                         logger.warning('twilio validation failure: %s not in possible sigs: %s',
                                        sig, expected_sigs)
@@ -787,6 +805,7 @@ def process_request(self, req, resp):
                     if qs:
                         path = path + '?' + qs
                     text = '%s %s %s %s' % (window, method, path, body)
+                    text = text if isinstance(text, bytes) else text.encode('utf8')
 
                     conn = db.connect()
                     cursor = conn.cursor()
@@ -797,6 +816,7 @@ def process_request(self, req, resp):
                     if row is None:
                         raise falcon.HTTPUnauthorized('Authentication failure: server')
                     key = self.fernet.decrypt(str(row[0]))
+                    key = key if isinstance(key, bytes) else key.encode('utf8')
                     req.context['user'] = row[1]
 
                     HMAC = hmac.new(key, text, hashlib.sha512)
@@ -856,6 +876,8 @@ def process_response(self, req, resp, resource, req_succeeded):
             )
             if not allow_headers:
                 allow_headers = '*'
+            if not allow:
+                allow = ''
 
             resp.set_headers((
                 ('Access-Control-Allow-Methods', allow),
@@ -867,7 +889,7 @@ def process_response(self, req, resp, resource, req_succeeded):
 def read_config_from_argv():
     import sys
     if len(sys.argv) < 2:
-        print 'Usage: %s CONFIG_FILE' % sys.argv[0]
+        print(('Usage: %s CONFIG_FILE' % sys.argv[0]))
         sys.exit(1)
 
     with open(sys.argv[1], 'r') as config_file:
@@ -950,7 +972,7 @@ def get_relay_server():
     config = read_config_from_argv()
     app = get_relay_app(config)
     server = config['server']
-    print 'LISTENING: %(host)s:%(port)d' % server
+    print(('LISTENING: %(host)s:%(port)d' % server))
     return WSGIServer((server['host'], server['port']), app)
 
 

src/iris_relay/client.py

@@ -7,7 +7,11 @@
 import hmac
 import hashlib
 import base64
-from urllib import urlencode
+# py 2 and 3 compatibility
+try:
+    from urllib import urlencode
+except ImportError:
+    from urllib.parse import urlencode
 from urllib3.connectionpool import HTTPConnectionPool
 
 
@@ -16,7 +20,7 @@ def __init__(self, host, port, user, api_key, version=0, **kwargs):
         super(IrisClient, self).__init__(host, port, **kwargs)
         self.version = version
         self.user = user
-        self.HMAC = hmac.new(api_key, '', hashlib.sha512)
+        self.HMAC = hmac.new(api_key if isinstance(api_key, bytes) else api_key.encode('utf8'), b'', hashlib.sha512)
         self.base_path = '/v%s/' % version if version is not None else '/'
 
     def post(self, endpoint, data, params=None, raw=False, headers=None):
@@ -35,10 +39,12 @@ def post(self, endpoint, data, params=None, raw=False, headers=None):
         if params:
             path = ''.join([path, '?', urlencode(params)])
         text = '%s %s %s %s' % (window, method, path, body)
+        text = text if isinstance(text, bytes) else text.encode('utf8')
         HMAC.update(text)
         digest = base64.urlsafe_b64encode(HMAC.digest())
 
-        hdrs['Authorization'] = 'hmac %s:' % self.user + digest
+        auth_header = 'hmac %s:' % self.user
+        hdrs['Authorization'] = auth_header if isinstance(auth_header, bytes) else auth_header.encode('utf8') + digest
 
         return self.urlopen(method, path, headers=hdrs, body=body)
 
@@ -51,11 +57,13 @@ def get(self, endpoint, params=None, raw=False):
         if params:
             path = ''.join([path, '?', urlencode(params)])
         text = '%s %s %s %s' % (window, method, path, body)
+        text = text if isinstance(text, bytes) else text.encode('utf8')
         HMAC.update(text)
         digest = base64.urlsafe_b64encode(HMAC.digest())
 
+        auth_header = 'hmac %s:' % self.user
         headers = {
             'Content-Type': 'application/json',
-            'Authorization': 'hmac %s:' % self.user + digest
+            'Authorization': auth_header if isinstance(auth_header, bytes) else auth_header.encode('utf8') + digest
         }
         return self.urlopen(method, path, headers=headers)

src/iris_relay/gmail.py

@@ -58,7 +58,9 @@ def process_message(message):
         # TODO(khrichar): support other content types
         mime_type = part.get('mimeType')
         if mime_type == 'text/plain':
-            content = urlsafe_b64decode(str(part.get('body', {}).get('data', '')))
+            encoded_content = part.get('body', {}).get('data', '')
+            encoded_content = encoded_content if isinstance(encoded_content, bytes) else encoded_content.encode('utf8')
+            content = urlsafe_b64decode(encoded_content)
             yield headers, content
         elif mime_type == 'text/html':
             logger.debug('ignore html mime type for message: %s', message)

test/e2etest.py

@@ -36,20 +36,22 @@ def test_twilio_phone_say_api():
     re = requests.get(base_url + 'twilio/calls/say?content=hello',
                       headers={'X-Twilio-Signature': signature})
     assert re.status_code == 200
-    assert re.content == (
+    content = (
         '<?xml version="1.0" encoding="UTF-8"?>'
         '<Response><Say language="en-US" voice="alice">hello</Say>'
         '</Response>')
+    assert re.content == content if isinstance(content, bytes) else content.encode('utf8')
     assert re.headers['content-type'] == 'application/xml'
 
     # Should have the same behavior on post
     re = requests.post(base_url + 'twilio/calls/say?content=hello',
                        headers={'X-Twilio-Signature': signature})
     assert re.status_code == 200
-    assert re.content == (
+    content = (
         '<?xml version="1.0" encoding="UTF-8"?>'
         '<Response><Say language="en-US" voice="alice">hello</Say>'
         '</Response>')
+    assert re.content == content if isinstance(content, bytes) else content.encode('utf8')
     assert re.headers['content-type'] == 'application/xml'
 
 
@@ -67,7 +69,7 @@ def test_twilio_phone_gather_api():
         },
         headers={'X-Twilio-Signature': 'EgL9vRByfCmVTsAcYRgq3e+nBHw='})
     assert re.status_code == 200
-    assert re.content == (
+    content = (
         '<?xml version="1.0" encoding="UTF-8"?><Response>'
         '<Pause length="2" />'
         '<Say language="en-US" voice="alice">Press pound for menu.</Say>'
@@ -81,6 +83,7 @@ def test_twilio_phone_gather_api():
         ' numDigits="1"><Say language="en-US" voice="alice">bar</Say>'
         '</Gather></Response>'
     ) % host
+    assert re.content == content if isinstance(content, bytes) else content.encode('utf8')
     assert re.headers['content-type'] == 'application/xml'
 
 
@@ -97,7 +100,7 @@ def test_twilio_phone_gather_api_batch_message_id():
         base_url + 'twilio/calls/gather', params=params,
         headers={'X-Twilio-Signature': 'y4SPekGJdZ1oH1k/UHFdf29epbo='})
     assert re.status_code == 200
-    assert re.content == (
+    content = (
         '<?xml version="1.0" encoding="UTF-8"?><Response>'
         '<Pause length="2" />'
         '<Say language="en-US" voice="alice">Press pound for menu.</Say>'
@@ -112,6 +115,7 @@ def test_twilio_phone_gather_api_batch_message_id():
         '<Say language="en-US" loop="3" voice="alice">Press 2 to claim.</Say>'
         '</Gather></Response>'
     ) % (host, fake_batch_id)
+    assert re.content == content if isinstance(content, bytes) else content.encode('utf8')
     assert re.headers['content-type'] == 'application/xml'
 
     params['message_id'] = ['arbitrary text']
@@ -171,7 +175,7 @@ def test_health():
     """
     re = requests.get(host + '/healthcheck')
     assert re.status_code == 200
-    assert re.content == 'GOOD'
+    assert re.content == b'GOOD'
 
 
 def test_gmail_verification():
@@ -182,4 +186,4 @@ def test_gmail_verification():
     # path
     re = requests.get(host + '/googleabcdefg.html')
     assert re.status_code == 200
-    assert re.content == 'google-site-verification: googleabcdefg.html'
+    assert re.content == b'google-site-verification: googleabcdefg.html'

test/test_gmail.py

@@ -44,6 +44,7 @@ def test_is_pointless_messages():
 def test_process_message_text_plain():
     fake_headers = [{u'name': u'Content-Type', u'value': u'text/plain; charset="us-ascii"'}]
     fake_content = 'hello'
+    fake_content = fake_content if isinstance(fake_content, bytes) else fake_content.encode('utf8')
     fake_message = {'payload': {
         'headers': fake_headers,
         'parts': [{'mimeType': 'text/plain', 'body': {'data': urlsafe_b64encode(fake_content)}}]
@@ -62,6 +63,7 @@ def test_process_message_multipart():
         'value': 'multipart/alternative; boundary="===============3481026495533768394=="'
     }]
     fake_content = 'hello'
+    fake_content = fake_content if isinstance(fake_content, bytes) else fake_content.encode('utf8')
     fake_message = {
         'payload': {
             'headers': fake_headers,

tools/list_gmail.py

@@ -8,6 +8,6 @@
 config = read_config_from_argv()
 gmclient = Gmail(config.get('gmail'), config.get('proxy'))
 
-print 'Fetching unread messages...'
+print('Fetching unread messages...')
 for msg_id_gmail, headers, body in gmclient.list_unread_message():
     print({'body': body, 'headers': headers})