feat: add timeout and retry_limit in loader_kwargs (#865 #831)

ScrapeGraphAI · Jan 6, 2025 · 21147c4 · 21147c4
1 parent 2a15581
commit 21147c4
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/scrapegraphai/docloaders/chromium.py b/scrapegraphai/docloaders/chromium.py
@@ -35,7 +35,7 @@ def __init__(
         storage_state: Optional[str] = None,
         browser_name: str = "chromium",  #default chromium
         retry_limit: int = 1,
-        timeout: int = 10,
+        timeout: int = 60,
         **kwargs: Any,
     ):
         """Initialize the loader with a list of URL paths.
@@ -69,8 +69,8 @@ def __init__(
         self.requires_js_support = requires_js_support
         self.storage_state = storage_state
         self.browser_name = browser_name
-        self.retry_limit = retry_limit
-        self.timeout = timeout
+        self.retry_limit = kwargs.get("retry_limit", retry_limit)
+        self.timeout = kwargs.get("timeout", timeout)
 
     async def scrape(self, url:str) -> str:
         if self.backend == "playwright":