Issue #105: Use "with" with requests connections

Nekmo · Apr 20, 2022 · 771b9a7 · 771b9a7
1 parent 0563d82
commit 771b9a7
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 6 deletions.
diff --git a/dirhunt/processors.py b/dirhunt/processors.py
@@ -89,7 +89,7 @@ def search_index_files(self):
             future = self.crawler_url.crawler.add_url(
                 CrawlerUrl(crawler, url, self.crawler_url.depth - 1, self, None, 'document',
                            timeout=self.crawler_url.timeout), True)
-            if self.crawler_url.crawler.closing:
+            if self.crawler_url.crawler.closing or future is None:
                 return
             result = future.result()
             if result.exists:

diff --git a/dirhunt/tests/test_crawler.py b/dirhunt/tests/test_crawler.py
@@ -59,7 +59,7 @@ def test_resume(self, m1, m2, m3):
         crawler.resume(crawler.get_resume_file())
         m3.assert_called_once()
         m2.assert_called_once()
-        m1.assert_called_once_with(REPORT_DATA['processing'][0])
+        m1.assert_called_once_with(REPORT_DATA['processing'][0], lock=False)
 
     def test_print_results_limit(self):
         crawler = self.get_crawler(limit=1)

diff --git a/dirhunt/tests/test_crawler_url.py b/dirhunt/tests/test_crawler_url.py
@@ -12,7 +12,7 @@
 class TestCrawlerUrl(CrawlerTestBase, unittest.TestCase):
     def test_start(self):
         crawler = self.get_crawler()
-        crawler.closing = True
+        crawler.closing = False
         crawler_url = CrawlerUrl(crawler, self.url)
         crawler.processing[self.url] = crawler_url
         with requests_mock.mock() as m:
@@ -35,9 +35,11 @@ def test_session_exception(self, req_mock):
     def test_session_read_exception(self):
         crawler = self.get_crawler()
         crawler.sessions = Mock()
-        crawler.sessions.get_session.return_value.get.return_value.status_code = 200
-        crawler.sessions.get_session.return_value.get.return_value.raw.read.side_effect = \
-            requests.exceptions.ConnectTimeout()
+        crawler.sessions.get_session.return_value.get.return_value.__enter__ = Mock(**{
+            'return_value.status_code': 200,
+            'return_value.raw.read.side_effect': requests.exceptions.ConnectTimeout(),
+        })
+        crawler.sessions.get_session.return_value.get.return_value.__exit__ = Mock()
         with patch('dirhunt.crawler_url.CrawlerUrl.close') as m:
             crawler_url = CrawlerUrl(crawler, self.url)
             self.assertEqual(crawler_url.start(), crawler_url)