🔨 Profile api server

services/api-server/README.md

@@ -36,6 +36,11 @@ will start the api-server in development-mode together with a postgres db initia
 - http://127.0.0.1:8000/docs: redoc documentation
 - http://127.0.0.1:8000/dev/docs: swagger type of documentation
 
+### Profiling requests to the api server
+When in development mode (the environment variable `API_SERVER_DEV_FEATURES_ENABLED` is =1 in the running container) one can profile calls to the API server directly from the client side. On the server, the profiling is done using [Pyinstrument](https://github.com/joerick/pyinstrument). If we have our request in the form of a curl command, one simply adds the custom header `x-profile-api-server:true` to the command to the request, in which case the profile is received under the `profile` key of the response body. This makes it easy to visualise the profiling report directly in bash:
+```bash
+<curl_command> -H 'x-profile-api-server: true' | jq -r .profile
+```
 
 ## Clients
 

services/api-server/requirements/_tools.in

@@ -7,3 +7,4 @@
 change_case # for tools/get_api
 jinja2 # for tools/get_api
 watchdog[watchmedo]
+pyinstrument

services/api-server/requirements/_tools.txt

@@ -72,6 +72,10 @@ platformdirs==3.10.0
     #   virtualenv
 pre-commit==3.3.3
     # via -r requirements/../../../requirements/devenv.txt
+pyinstrument==4.5.0
+    # via
+    #   -c requirements/_base.txt
+    #   -r requirements/_tools.in
 pylint==2.17.5
     # via -r requirements/../../../requirements/devenv.txt
 pyproject-hooks==1.0.0

services/api-server/src/simcore_service_api_server/core/application.py

@@ -1,4 +1,6 @@
+import json
 import logging
+import os
 
 from fastapi import FastAPI
 from fastapi.exceptions import RequestValidationError
@@ -23,9 +25,61 @@
 from .openapi import override_openapi_method, use_route_names_as_operation_ids
 from .settings import ApplicationSettings
 
+if os.environ.get("API_SERVER_DEV_FEATURES_ENABLED"):
+    from pyinstrument import Profiler
+    from starlette.requests import Request
+
+
 _logger = logging.getLogger(__name__)
 
 
+class ApiServerProfilerMiddleware:
+    """Following
+    https://www.starlette.io/middleware/#cleanup-and-error-handling
+    https://www.starlette.io/middleware/#reusing-starlette-components
+    https://fastapi.tiangolo.com/advanced/middleware/#advanced-middleware
+    """
+
+    def __init__(self, app: FastAPI):
+        self._app: FastAPI = app
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] != "http":
+            await self._app(scope, receive, send)
+            return
+
+        profiler = Profiler(async_mode="enabled")
+        request: Request = Request(scope)
+        headers = dict(request.headers)
+        if "x-profile-api-server" in headers:
+            del headers["x-profile-api-server"]
+            scope["headers"] = [
+                (k.encode("utf8"), v.encode("utf8")) for k, v in headers.items()
+            ]
+            profiler.start()
+
+        async def send_wrapper(message):
+            if profiler.is_running:
+                profiler.stop()
+            if profiler.last_session:
+                body: bytes = json.dumps(
+                    {"profile": profiler.output_text(unicode=True, color=True)}
+                ).encode("utf8")
+                if message["type"] == "http.response.start":
+                    for ii, header in enumerate(message["headers"]):
+                        key, _ = header
+                        if key.decode("utf8") == "content-length":
+                            message["headers"][ii] = (
+                                key,
+                                str(len(body)).encode("utf8"),
+                            )
+                elif message["type"] == "http.response.body":
+                    message = {"type": "http.response.body", "body": body}
+            await send(message)
+
+        await self._app(scope, receive, send_wrapper)
+
+
 def _label_info_with_state(settings: ApplicationSettings, title: str, version: str):
     labels = []
     if settings.API_SERVER_DEV_FEATURES_ENABLED:
@@ -113,6 +167,8 @@ def init_app(settings: ApplicationSettings | None = None) -> FastAPI:
             else None,
         ),
     )
+    if settings.API_SERVER_DEV_FEATURES_ENABLED:
+        app.add_middleware(ApiServerProfilerMiddleware)
 
     # routing