Merge pull request #104 from soxoj/speed-up

Accelerated module import time

socid_extractor/main.py

@@ -1,8 +1,5 @@
 import logging
 from http.cookies import SimpleCookie
-from bs4 import BeautifulSoup as bs
-
-import requests
 
 from .schemes import *
 from .postprocessor import POSTPROCESSORS
@@ -21,6 +18,8 @@ def parse(url, cookies_str='', timeout=3, headers={}):
     req_headers = dict(HEADERS)
     req_headers.update(headers)
     logging.debug(req_headers)
+
+    import requests
     page = requests.get(url, headers=req_headers, cookies=cookies, allow_redirects=True, timeout=(timeout, timeout))
     logging.debug('Server response: \'%s\'', page.text)
     logging.debug('Status code: %d', page.status_code)
@@ -129,6 +128,7 @@ def extract(page):
                     values = map_fields(scheme_data, transformed_data)
 
         if use_html_parser:
+            from bs4 import BeautifulSoup as bs
             soup = bs(page, 'html.parser')
             for name, get_field in scheme_data['fields'].items():
                 try:

socid_extractor/postprocessor.py

@@ -1,5 +1,4 @@
 import re
-import requests
 
 
 class Gravatar:
@@ -28,6 +27,7 @@ def make_en_url(self):
         return f'https://en.gravatar.com/{self.email_hash}'
 
     def get_username(self):
+        import requests
         gravatar_account_location = requests.head(self.make_en_url())
         username = gravatar_account_location.headers.get('location', '').strip('/')
         if username == 'profiles/no-such-user':

socid_extractor/utils.py

@@ -1,15 +1,12 @@
-from http.cookiejar import MozillaCookieJar
-from http.cookies import Morsel
-
 import logging
 import math
 import re
-import requests
 from datetime import datetime
 from http.cookies import SimpleCookie
 
 
 def import_cookiejar(filename):
+    from http.cookiejar import MozillaCookieJar
     cookies_obj = MozillaCookieJar(filename)
     cookies_obj.load(ignore_discard=True, ignore_expires=True)
     cookies = {}
@@ -111,5 +108,7 @@ def extract_periscope_uid(text):
     return userId.group(1)
 
 def get_mymail_uid(username):
+    # TODO: move to external function
+    import requests
     req = requests.get('http://appsmail.ru/platform/mail/' + username)
     return req.json()['uid']