Merge pull request #16 from Affirm/cert_verify

add option to verify certs for http requests

luigi/rpc.py

@@ -76,6 +76,26 @@ def fetch(self, full_url, body, timeout):
         return urlopen(full_url, body, timeout).read().decode('utf-8')
 
 
+class BobaPKIHTTPAdapter(requests.adapters.HTTPAdapter):
+    """HTTP adapter which disables hostname validation on HTTPS connections.
+
+    Copied from $ATT/rpc2/affirm/rpc2/transport/http.py
+    """
+
+    def init_poolmanager(self, *args, **kwargs):
+        super(BobaPKIHTTPAdapter, self).init_poolmanager(assert_hostname=False, *args, **kwargs)
+
+
+def get_requests_session():
+    session = requests.Session()
+    cert_verify = os.environ.get('BOBAPKI_CACERT_VERIFY', None)
+    if cert_verify is not None:
+        session.mount('https://', BobaPKIHTTPAdapter())
+        session.verify = cert_verify
+
+    return session
+
+
 class RequestsFetcher(object):
     def __init__(self, session):
         from requests import exceptions as requests_exceptions
@@ -87,7 +107,7 @@ def check_pid(self):
         # if the process id change changed from when the session was created
         # a new session needs to be setup since requests isn't multiprocessing safe.
         if os.getpid() != self.process_id:
-            self.session = requests.Session()
+            self.session = get_requests_session()
             self.process_id = os.getpid()
 
     def fetch(self, full_url, body, timeout):
@@ -118,7 +138,7 @@ def __init__(self, url='http://localhost:8082/', connect_timeout=None):
         self._rpc_retry_wait = config.getint('core', 'rpc-retry-wait', 30)
 
         if HAS_REQUESTS:
-            self._fetcher = RequestsFetcher(requests.Session())
+            self._fetcher = RequestsFetcher(get_requests_session())
         else:
             self._fetcher = URLLibFetcher()
 

setup.py

@@ -54,7 +54,7 @@ def get_static_files(path):
 
 setup(
     name='luigi',
-    version='2.7.5.affirm.1.4.0',
+    version='2.7.5+affirm.1.4.2',
     description='Workflow mgmgt + task scheduling + dependency resolution',
     long_description=long_description,
     author='The Luigi Authors',