Increase client timeouts and tweak response_fields behaviour

optimade/client/cli.py

@@ -73,6 +73,11 @@
     default=None,
     nargs=-1,
 )
+@click.option(
+    "--http-timeout",
+    type=float,
+    help="The timeout to use for each HTTP request.",
+)
 def get(
     use_async,
     filter,
@@ -87,6 +92,7 @@ def get(
     include_providers,
     exclude_providers,
     exclude_databases,
+    http_timeout,
 ):
     return _get(
         use_async,
@@ -102,6 +108,7 @@ def get(
         include_providers,
         exclude_providers,
         exclude_databases,
+        http_timeout,
     )
 
 
@@ -119,6 +126,7 @@ def _get(
     include_providers,
     exclude_providers,
     exclude_databases,
+    http_timeout,
     **kwargs,
 ):
     if output_file:
@@ -130,7 +138,7 @@ def _get(
                 f"Desired output file {output_file} already exists, not overwriting."
             )
 
-    client = OptimadeClient(
+    args = dict(
         base_urls=base_url,
         use_async=use_async,
         max_results_per_provider=max_results_per_provider,
@@ -143,6 +151,15 @@ def _get(
         exclude_databases=set(_.strip() for _ in exclude_databases.split(","))
         if exclude_databases
         else None,
+    )
+
+    # Only set http timeout if its not null to avoid overwriting or duplicating the
+    # default value set on the OptimadeClient class
+    if http_timeout:
+        args["http_timeout"] = http_timeout
+
+    client = OptimadeClient(
+        **args,
         **kwargs,
     )
     if response_fields:

optimade/client/client.py

@@ -76,7 +76,7 @@ class OptimadeClient:
     headers: Dict = {"User-Agent": f"optimade-python-tools/{__version__}"}
     """Additional HTTP headers."""
 
-    http_timeout: int
+    http_timeout: httpx.Timeout = httpx.Timeout(10.0, read=1000.0)
     """The timeout to use for each HTTP request."""
 
     max_attempts: int
@@ -106,7 +106,7 @@ def __init__(
         base_urls: Optional[Union[str, Iterable[str]]] = None,
         max_results_per_provider: int = 1000,
         headers: Optional[Dict] = None,
-        http_timeout: int = 10,
+        http_timeout: Optional[Union[httpx.Timeout, float]] = None,
         max_attempts: int = 5,
         use_async: bool = True,
         exclude_providers: Optional[List[str]] = None,
@@ -121,7 +121,9 @@ def __init__(
             max_results_per_provider: The maximum number of results to download
                 from each provider.
             headers: Any additional HTTP headers to use for the queries.
-            http_timeout: The HTTP timeout to use per request.
+            http_timeout: The timeout to use per request. Defaults to 10
+                seconds with 1000 seconds for reads specifically. Overriding this value
+                will replace all timeouts (connect, read, write and pool) with this value.
             max_attempts: The maximum number of times to repeat a failing query.
             use_async: Whether or not to make all requests asynchronously.
             exclude_providers: A set or collection of provider IDs to exclude from queries.
@@ -165,7 +167,12 @@ def __init__(
         if headers:
             self.headers.update(headers)
 
-        self.http_timeout = http_timeout
+        if http_timeout:
+            if isinstance(http_timeout, httpx.Timeout):
+                self.http_timeout = http_timeout
+            else:
+                self.http_timeout = httpx.Timeout(http_timeout)
+
         self.max_attempts = max_attempts
 
         self.use_async = use_async
@@ -309,7 +316,7 @@ def count(
                 endpoint,
                 page_limit=1,
                 paginate=False,
-                response_fields=None,
+                response_fields=[],
                 sort=None,
             )
             count_results = {}
@@ -321,7 +328,7 @@ def count(
 
                 if count_results[base_url] is None:
                     self._progress.print(
-                        f"Warning: {base_url} did not return a value for `meta->data_returned`, unable to count results."
+                        f"Warning: {base_url} did not return a value for `meta->data_returned`, unable to count results. Full response: {results[base_url]}"
                     )
 
             self.count_results[endpoint][filter] = count_results
@@ -429,7 +436,9 @@ def get_one(
             )
         except (RuntimeError, httpx.TimeoutException, json.JSONDecodeError) as exc:
             error_query_results = QueryResults()
-            error_query_results.errors = [str(exc)]
+            error_query_results.errors = [
+                f"{exc.__class__.__name__}: {str(exc.args[0])}"
+            ]
             self._progress.print(
                 f"[red]Error[/red]: Provider {str(base_url)!r} returned: [red i]{exc}[/red i]"
             )
@@ -574,9 +583,11 @@ async def get_one_async(
             Exception,
         ) as exc:
             error_query_results = QueryResults()
-            error_query_results.errors = [str(exc)]
+            error_query_results.errors = [
+                f"{exc.__class__.__name__}: {str(exc.args[0])}"
+            ]
             self._progress.print(
-                f"[red]Error[/red]: Provider {str(base_url)!r} returned: [red i]{exc}[/red i]"
+                f"[red]Error[/red]: Provider {str(base_url)!r} returned: [red i]{error_query_results.errors}[/red i]"
             )
             return {base_url: error_query_results}
 
@@ -765,8 +776,12 @@ def _build_url(
 
         if filter:
             _filter = f"filter={filter}"
-        if response_fields:
-            _response_fields = f'response_fields={",".join(response_fields)}'
+        if response_fields is not None:
+            # If we have requested no response fields (e.g., in the case of --count) then just ask for IDs
+            if len(response_fields) == 0:
+                _response_fields = "response_fields=id"
+            else:
+                _response_fields = f'response_fields={",".join(response_fields)}'
         if page_limit:
             _page_limit = f"page_limit={page_limit}"
         if sort:

tests/server/test_client.py

@@ -157,6 +157,8 @@ def test_client_sort(http_client, use_async):
 
 @pytest.mark.parametrize("use_async", [False])
 def test_command_line_client(http_client, use_async, capsys):
+    import httpx
+
     from optimade.client.cli import _get
 
     args = dict(
@@ -174,6 +176,7 @@ def test_command_line_client(http_client, use_async, capsys):
         exclude_providers=None,
         exclude_databases=None,
         http_client=http_client,
+        http_timeout=httpx.Timeout(2.0),
     )
 
     # Test multi-provider query