feature/mx-1571 prepare rule implementations

.pre-commit-config.yaml

@@ -3,11 +3,11 @@ default_language_version:
   python: python3.11
 repos:
   - repo: https://github.com/psf/black
-    rev: 24.3.0
+    rev: 24.4.2
     hooks:
       - id: black
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.5
+    rev: v0.4.6
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
@@ -28,7 +28,7 @@ repos:
       - id: fix-byte-order-marker
         name: byte-order
   - repo: https://github.com/pdm-project/pdm
-    rev: 2.13.2
+    rev: 2.15.3
     hooks:
       - id: pdm-lock-check
         name: pdm

CHANGELOG.md

@@ -15,6 +15,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - updated graph connector for new queries
 - improved isolation of neo4j dependency
 - improved documentation and code-readability
+- move exception handling middleware to new module
+- change `identity_provider` default to `MEMORY`
+- add stop-gap code waiting to be resolved by mx-1596
+- migrate to latest `mex-common`
 
 ### Deprecated
 

mex/backend/exceptions.py

@@ -0,0 +1,55 @@
+from typing import Any
+
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel, ValidationError
+from starlette.requests import Request
+
+from mex.backend.transform import to_primitive
+from mex.common.logging import logger
+
+
+class DebuggingScope(BaseModel, extra="ignore"):
+    """Scope for debugging info of error responses."""
+
+    http_version: str
+    method: str
+    path: str
+    path_params: dict[str, Any]
+    query_string: str
+    scheme: str
+
+
+class DebuggingInfo(BaseModel):
+    """Debugging information for error responses."""
+
+    errors: list[dict[str, Any]]
+    scope: DebuggingScope
+
+
+class ErrorResponse(BaseModel):
+    """Response model for user and system errors."""
+
+    message: str
+    debug: DebuggingInfo
+
+
+def handle_uncaught_exception(request: Request, exc: Exception) -> JSONResponse:
+    """Handle uncaught errors and provide debugging info."""
+    logger.exception("Error %s", exc)
+    if isinstance(exc, ValidationError):
+        errors = [dict(error) for error in exc.errors()]
+        status_code = 400
+    else:
+        errors = [dict(type=type(exc).__name__)]
+        status_code = 500
+    return JSONResponse(
+        to_primitive(
+            ErrorResponse(
+                message=str(exc),
+                debug=DebuggingInfo(
+                    errors=errors, scope=DebuggingScope.model_validate(request.scope)
+                ),
+            )
+        ),
+        status_code,
+    )

mex/backend/graph/connector.py

@@ -71,7 +71,7 @@ def _check_connectivity_and_authentication(self) -> Result:
         query_builder = QueryBuilder.get()
         result = self.commit(query_builder.fetch_database_status())
         if (status := result["currentStatus"]) != "online":
-            raise MExError(f"Database is {status}.")
+            raise MExError(f"Database is {status}.") from None
         return result
 
     def _seed_constraints(self) -> list[Result]:

mex/backend/graph/models.py

@@ -61,7 +61,7 @@ def one_or_none(self) -> dict[str, Any] | None:
             case 0:
                 return None
             case _:
-                raise MultipleResultsFoundError
+                raise MultipleResultsFoundError from None
 
     def get_update_counters(self) -> dict[str, int]:
         """Return a summary of counters for operations the query triggered."""

mex/backend/main.py

@@ -6,22 +6,18 @@
 from fastapi import APIRouter, Depends, FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.openapi.utils import get_openapi
-from fastapi.responses import JSONResponse
-from pydantic import BaseModel, ValidationError
-from starlette.requests import Request
+from pydantic import BaseModel
 
+from mex.backend.exceptions import handle_uncaught_exception
 from mex.backend.extracted.main import router as extracted_router
 from mex.backend.identity.main import router as identity_router
 from mex.backend.ingest.main import router as ingest_router
 from mex.backend.logging import UVICORN_LOGGING_CONFIG
 from mex.backend.merged.main import router as merged_router
 from mex.backend.security import has_read_access, has_write_access
 from mex.backend.settings import BackendSettings
-from mex.backend.transform import to_primitive
 from mex.common.cli import entrypoint
-from mex.common.connector import ConnectorContext
-from mex.common.exceptions import MExError
-from mex.common.logging import logger
+from mex.common.connector import CONNECTOR_STORE
 from mex.common.types import Identifier
 from mex.common.types.identifier import MEX_ID_PATTERN
 
@@ -61,24 +57,11 @@ def create_openapi_schema() -> dict[str, Any]:
     return app.openapi_schema
 
 
-def close_connectors() -> None:
-    """Try to close all connectors in the current context."""
-    context = ConnectorContext.get()
-    for connector_type, connector in context.items():
-        try:
-            connector.close()
-        except Exception:
-            logger.exception("Error closing %s", connector_type)
-        else:
-            logger.info("Closed %s", connector_type)
-    context.clear()
-
-
 @asynccontextmanager
 async def lifespan(_: FastAPI) -> AsyncIterator[None]:
     """Async context manager to execute setup and teardown of the FastAPI app."""
     yield None
-    close_connectors()
+    CONNECTOR_STORE.reset()
 
 
 app = FastAPI(
@@ -116,20 +99,7 @@ def check_system_status() -> SystemStatus:
     return SystemStatus(status="ok")
 
 
-def handle_uncaught_exception(request: Request, exc: Exception) -> JSONResponse:
-    """Handle uncaught errors and provide some debugging clues."""
-    logger.exception("Error %s", exc)
-    if isinstance(exc, ValidationError):
-        errors: list[Any] = exc.errors()
-    else:
-        errors = [dict(type=type(exc).__name__)]
-    body = dict(message=str(exc), debug=dict(errors=errors))
-    return JSONResponse(to_primitive(body), 500)
-
-
 app.include_router(router)
-app.add_exception_handler(ValidationError, handle_uncaught_exception)
-app.add_exception_handler(MExError, handle_uncaught_exception)
 app.add_exception_handler(Exception, handle_uncaught_exception)
 app.add_middleware(
     CORSMiddleware,

mex/backend/security.py

@@ -12,7 +12,7 @@
 X_API_CREDENTIALS = HTTPBasic(auto_error=False)
 
 
-def __check_header_for_authorization_method(
+def _check_header_for_authorization_method(
     api_key: Annotated[str | None, Depends(X_API_KEY)] = None,
     credentials: Annotated[
         HTTPBasicCredentials | None, Depends(X_API_CREDENTIALS)
@@ -71,15 +71,15 @@ def has_write_access(
     Settings:
         check credentials in backend_user_database or backend_api_key_database
     """
-    __check_header_for_authorization_method(api_key, credentials, user_agent)
+    _check_header_for_authorization_method(api_key, credentials, user_agent)
 
     settings = BackendSettings.get()
     can_write = False
     if api_key:
         api_key_database = settings.backend_api_key_database
-        can_write = APIKey(api_key) in api_key_database.write
+        can_write = APIKey(api_key) in api_key_database["write"]
     elif credentials:
-        api_write_user_db = settings.backend_user_database.write
+        api_write_user_db = settings.backend_user_database["write"]
         user, pw = credentials.username, credentials.password.encode("utf-8")
         if api_write_user := api_write_user_db.get(user):
             can_write = compare_digest(
@@ -118,7 +118,7 @@ def has_read_access(
     Settings:
         check credentials in backend_user_database or backend_api_key_database
     """
-    __check_header_for_authorization_method(api_key, credentials, user_agent)
+    _check_header_for_authorization_method(api_key, credentials, user_agent)
 
     try:
         has_write_access(api_key, credentials)  # read access implied by write access
@@ -130,9 +130,9 @@ def has_read_access(
     can_read = False
     if api_key:
         api_key_database = settings.backend_api_key_database
-        can_read = APIKey(api_key) in api_key_database.read
+        can_read = APIKey(api_key) in api_key_database["read"]
     elif credentials:
-        api_read_user_db = settings.backend_user_database.read
+        api_read_user_db = settings.backend_user_database["read"]
         user, pw = credentials.username, credentials.password.encode("utf-8")
         if api_read_user := api_read_user_db.get(user):
             can_read = compare_digest(

mex/backend/settings.py

@@ -71,7 +71,7 @@ class BackendSettings(BaseSettings):
         validation_alias="MEX_BACKEND_API_USER_DATABASE",
     )
     identity_provider: IdentityProvider | BackendIdentityProvider = Field(
-        BackendIdentityProvider.GRAPH,
+        IdentityProvider.MEMORY,
         description="Provider to assign stableTargetIds to new model instances.",
         validation_alias="MEX_IDENTITY_PROVIDER",
     )  # type: ignore[assignment]

mex/backend/transform.py

@@ -8,6 +8,7 @@
 
 JSON_ENCODERS: Final[dict[type, Callable[[Any], str]]] = {
     Enum: lambda obj: str(obj.value),
+    Exception: lambda obj: str(obj),
     Identifier: lambda obj: str(obj),
     TemporalEntity: lambda obj: str(obj),
 }

mex/backend/types.py

@@ -1,10 +1,9 @@
 from enum import Enum, EnumMeta, _EnumDict
-from typing import Literal
+from typing import Literal, cast
 
 from pydantic import SecretStr
 
 from mex.common.models import (
-    BASE_MODEL_CLASSES_BY_NAME,
     EXTRACTED_MODEL_CLASSES_BY_NAME,
     MERGED_MODEL_CLASSES_BY_NAME,
     BaseModel,
@@ -39,13 +38,21 @@ class APIKeyDatabase(BaseModel):
     read: list[APIKey] = []
     write: list[APIKey] = []
 
+    def __getitem__(self, key: str) -> list[APIKey]:  # stop-gap: MX-1596
+        """Return an attribute in indexing syntax."""
+        return cast(list[APIKey], getattr(self, key))
+
 
 class APIUserDatabase(BaseModel):
     """Database containing usernames and passwords for backend API."""
 
     read: dict[str, APIUserPassword] = {}
     write: dict[str, APIUserPassword] = {}
 
+    def __getitem__(self, key: str) -> dict[str, APIUserPassword]:  # stop-gap: MX-1596
+        """Return an attribute in indexing syntax."""
+        return cast(dict[str, APIUserPassword], getattr(self, key))
+
 
 class BackendIdentityProvider(Enum):
     """Identity providers implemented by mex-backend."""
@@ -68,7 +75,7 @@ def __new__(
 class UnprefixedType(Enum, metaclass=DynamicStrEnum):
     """Enumeration of possible types without any prefix."""
 
-    __names__ = list(m.removeprefix("Base") for m in BASE_MODEL_CLASSES_BY_NAME)
+    __names__ = [m.removeprefix("Extracted") for m in EXTRACTED_MODEL_CLASSES_BY_NAME]
 
 
 class ExtractedType(Enum, metaclass=DynamicStrEnum):