diff --git a/dirhunt/sources/commoncrawl.py b/dirhunt/sources/commoncrawl.py index f490130..0bcd9b7 100644 --- a/dirhunt/sources/commoncrawl.py +++ b/dirhunt/sources/commoncrawl.py @@ -31,12 +31,6 @@ def callback(self, domain): if not latest_crawl_index: return session = Sessions().get_session() - response = session.get( - latest_crawl_index, - params={'url': '*.{}'.format(domain), 'output': 'json'}, - timeout=TIMEOUT, - stream=True - ) try: with session.get(latest_crawl_index, params={'url': '*.{}'.format(domain), 'output': 'json'}, timeout=TIMEOUT, stream=True) as response: diff --git a/dirhunt/sources/wayback.py b/dirhunt/sources/wayback.py index 2bc5b36..cdf29bb 100644 --- a/dirhunt/sources/wayback.py +++ b/dirhunt/sources/wayback.py @@ -12,6 +12,7 @@ 'limit': '10000' } DEFAULT_ENCODING = 'utf-8' +TIMEOUT = 10 class Wayback(Source):