Format and improve Alembic migration generation (#396)

### Description

- import `app.type.sqlalchemy.TZDateTime` in migration files for custom
DATETIME objects
 - lint and format migration files
- don't put the migration revision and down_revision in the file
docstring as we usually forget to update them

### Checklist

- [x] All tests passing
- [x] Extended the documentation, if necessary

README.md

@@ -118,6 +118,38 @@ POSTGRES_DB = "hyperion"
 uvicorn app.main:app --reload
 ```
 
+## Use Alembic migrations
+
+The project use Alembic migrations to manage database structure evolutions.
+
+When the database does not exist, SQLAlchemy will create a new database with an up to date structure. When the database already exist, migrations must be run to update the structure.
+
+### Run migrations
+
+These [migration files](./migrations/versions/) are automatically run before Hyperion startup.
+
+They can also be run manually using the following command:
+
+```bash
+alembic upgrade head
+```
+
+> If you want to force Alembic to consider your database structure is up to date, you can use the following command:
+>
+> ```bash
+> alembic stamp head
+> ```
+
+### Write migration files
+
+To create a new migration file, use the following command:
+
+```bash
+alembic revision --autogenerate -m "Your message"
+```
+
+Files must be names with the following convention: `number-message.py
+
 ## OpenAPI specification
 
 API endpoints are parsed following the OpenAPI specifications at `http://127.0.0.1:8000/openapi.json`.

alembic.ini

@@ -66,6 +66,17 @@ version_path_separator = os  # Use os.pathsep. Default configuration used for ne
 # black.entrypoint = black
 # black.options = -l 79 REVISION_SCRIPT_FILENAME
 
+# Lint and format using Ruff
+hooks=ruff_check, ruff_format
+
+ruff_check.type = exec
+ruff_check.executable = ruff
+ruff_check.options = check --fix REVISION_SCRIPT_FILENAME
+
+ruff_format.type = exec
+ruff_format.executable = ruff
+ruff_format.options = format REVISION_SCRIPT_FILENAME
+
 # Logging configuration
 [loggers]
 keys = root,sqlalchemy,alembic

app/core/auth/endpoints_auth.py

@@ -36,9 +36,9 @@
     get_token_data,
     get_user_from_token_with_scopes,
 )
+from app.types.scopes_type import ScopeType
 from app.utils.auth.providers import BaseAuthClient
 from app.utils.tools import is_user_member_of_an_allowed_group
-from app.utils.types.scopes_type import ScopeType
 
 router = APIRouter(tags=["Auth"])
 

app/core/auth/models_auth.py

@@ -4,7 +4,7 @@
 from sqlalchemy.orm import Mapped, mapped_column
 
 from app.database import Base
-from app.utils.types.datetime import TZDateTime
+from app.types.sqlalchemy import TZDateTime
 
 
 class AuthorizationCode(Base):

app/core/models_core.py

@@ -6,8 +6,8 @@
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from app.database import Base
-from app.utils.types.datetime import TZDateTime
-from app.utils.types.floors_type import FloorsType
+from app.types.floors_type import FloorsType
+from app.types.sqlalchemy import TZDateTime
 
 
 class CoreMembership(Base):

app/core/notification/models_notification.py

@@ -5,7 +5,7 @@
 
 from app.core.notification.notification_types import Topic
 from app.database import Base
-from app.utils.types.datetime import TZDateTime
+from app.types.sqlalchemy import TZDateTime
 
 
 class Message(Base):

app/core/schemas_core.py

@@ -6,9 +6,9 @@
 from pydantic.functional_validators import field_validator
 
 from app.core.groups.groups_type import AccountType
+from app.types.floors_type import FloorsType
 from app.utils import validators
 from app.utils.examples import examples_core
-from app.utils.types.floors_type import FloorsType
 
 
 class CoreInformation(BaseModel):

app/dependencies.py

@@ -29,10 +29,10 @@ async def get_users(db: AsyncSession = Depends(get_db)):
 from app.core.config import Settings
 from app.core.groups.groups_type import GroupType
 from app.core.users import cruds_users
+from app.types.scopes_type import ScopeType
 from app.utils.communication.notifications import NotificationManager, NotificationTool
 from app.utils.redis import connect
 from app.utils.tools import is_user_member_of_an_allowed_group
-from app.utils.types.scopes_type import ScopeType
 
 # We could maybe use hyperion.security
 hyperion_access_logger = logging.getLogger("hyperion.access")

app/modules/advert/models_advert.py

@@ -4,7 +4,7 @@
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from app.database import Base
-from app.utils.types.datetime import TZDateTime
+from app.types.sqlalchemy import TZDateTime
 
 
 class Advertiser(Base):

app/modules/amap/models_amap.py

@@ -8,7 +8,7 @@
 from app.core.models_core import CoreUser
 from app.database import Base
 from app.modules.amap.types_amap import AmapSlotType, DeliveryStatusType
-from app.utils.types.datetime import TZDateTime
+from app.types.sqlalchemy import TZDateTime
 
 
 class AmapOrderContent(Base):

app/modules/booking/models_booking.py

@@ -7,7 +7,7 @@
 
 from app.core.models_core import CoreUser
 from app.database import Base
-from app.utils.types.datetime import TZDateTime
+from app.types.sqlalchemy import TZDateTime
 
 
 class Manager(Base):

app/modules/calendar/models_calendar.py

@@ -6,7 +6,7 @@
 from app.core.models_core import CoreUser
 from app.database import Base
 from app.modules.calendar.types_calendar import CalendarEventType
-from app.utils.types.datetime import TZDateTime
+from app.types.sqlalchemy import TZDateTime
 
 
 class Event(Base):

app/modules/cinema/models_cinema.py

@@ -4,7 +4,7 @@
 from sqlalchemy.orm import Mapped, mapped_column
 
 from app.database import Base
-from app.utils.types.datetime import TZDateTime
+from app.types.sqlalchemy import TZDateTime
 
 
 class Session(Base):

app/modules/recommendation/models_recommendation.py

@@ -4,7 +4,7 @@
 from sqlalchemy.orm import Mapped, mapped_column
 
 from app.database import Base
-from app.utils.types.datetime import TZDateTime
+from app.types.sqlalchemy import TZDateTime
 
 
 class Recommendation(Base):

app/utils/types/datetime.py → app/types/sqlalchemy.py

@@ -5,8 +5,13 @@
 
 
 class TZDateTime(TypeDecorator):
-    # see https://docs.sqlalchemy.org/en/20/core/custom_types.html#store-timezone-aware-timestamps-as-timezone-naive-utc
-    # We use this custom type because sqlite doesn't support datetime with timezone
+    """
+    Custom SQLAlchemy type for storing timezone-aware timestamps as timezone-naive UTC timestamps.
+    We use this custom type because sqlite doesn't support datetime with timezone
+    See https://docs.sqlalchemy.org/en/20/core/custom_types.html#store-timezone-aware-timestamps-as-timezone-naive-utc
+    """
+
+    # Changing this type may break existing migrations. You may prefer to create a new version of the type instead.
 
     impl = DateTime
     cache_ok = True

app/utils/auth/providers.py

@@ -5,8 +5,8 @@
 
 from app.core import models_core
 from app.core.groups.groups_type import GroupType
+from app.types.scopes_type import ScopeType
 from app.utils.tools import get_display_name, is_user_member_of_an_allowed_group
-from app.utils.types.scopes_type import ScopeType
 
 
 class BaseAuthClient:
@@ -22,7 +22,7 @@ class BaseAuthClient:
     ########################################################
 
     # Set of scopes the auth client is authorized to grant when issuing an access token.
-    # See app.utils.types.scopes_type.ScopeType for possible values
+    # See app.types.scopes_type.ScopeType for possible values
     # If the client always send a specific scope (ex: `name`), you may add it to the allowed scopes as a string (ex: `"name"`)
     # These "string" scopes won't have any effect for Hyperion but won't raise a warning when asked by the client
     # WARNING: to be able to use openid connect, `ScopeType.openid` should always be allowed
@@ -90,7 +90,7 @@ class AppAuthClient(BaseAuthClient):
     """
 
     # Set of scopes the auth client is authorized to grant when issuing an access token.
-    # See app.utils.types.scopes_type.ScopeType for possible values
+    # See app.types.scopes_type.ScopeType for possible values
     # WARNING: to be able to use openid connect, `ScopeType.openid` should always be allowed
     allowed_scopes: set[ScopeType | str] = {ScopeType.API}
     # Restrict the authentication to this client to specific Hyperion groups.
@@ -104,14 +104,14 @@ class PostmanAuthClient(BaseAuthClient):
     """
 
     # Set of scopes the auth client is authorized to grant when issuing an access token.
-    # See app.utils.types.scopes_type.ScopeType for possible values
+    # See app.types.scopes_type.ScopeType for possible values
     # WARNING: to be able to use openid connect, `ScopeType.openid` should always be allowed
     allowed_scopes: set[ScopeType | str] = {ScopeType.API}
 
 
 class NextcloudAuthClient(BaseAuthClient):
     # Set of scopes the auth client is authorized to grant when issuing an access token.
-    # See app.utils.types.scopes_type.ScopeType for possible values
+    # See app.types.scopes_type.ScopeType for possible values
     allowed_scopes: set[ScopeType | str] = {ScopeType.openid}
 
     # For Nextcloud:
@@ -142,7 +142,7 @@ def get_userinfo(cls, user: models_core.CoreUser):
 
 class PiwigoAuthClient(BaseAuthClient):
     # Set of scopes the auth client is authorized to grant when issuing an access token.
-    # See app.utils.types.scopes_type.ScopeType for possible values
+    # See app.types.scopes_type.ScopeType for possible values
     # WARNING: to be able to use openid connect, `ScopeType.openid` should always be allowed
     allowed_scopes: set[ScopeType | str] = {ScopeType.openid}
     # Restrict the authentication to this client to specific Hyperion groups.
@@ -183,7 +183,7 @@ def get_userinfo(self, user: models_core.CoreUser) -> dict[str, Any]:
 
 class HedgeDocAuthClient(BaseAuthClient):
     # Set of scopes the auth client is authorized to grant when issuing an access token.
-    # See app.utils.types.scopes_type.ScopeType for possible values
+    # See app.types.scopes_type.ScopeType for possible values
     allowed_scopes: set[ScopeType | str] = {ScopeType.profile}
 
     @classmethod
@@ -199,7 +199,7 @@ class WikijsAuthClient(BaseAuthClient):
     # https://github.com/requarks/wiki/blob/main/server/modules/authentication/oidc/definition.yml
 
     # Set of scopes the auth client is authorized to grant when issuing an access token.
-    # See app.utils.types.scopes_type.ScopeType for possible values
+    # See app.types.scopes_type.ScopeType for possible values
     allowed_scopes: set[ScopeType | str] = {ScopeType.openid, ScopeType.profile}
 
     @classmethod
@@ -218,7 +218,7 @@ def get_userinfo(cls, user: models_core.CoreUser):
 
 class SynapseAuthClient(BaseAuthClient):
     # Set of scopes the auth client is authorized to grant when issuing an access token.
-    # See app.utils.types.scopes_type.ScopeType for possible values
+    # See app.types.scopes_type.ScopeType for possible values
     allowed_scopes: set[ScopeType | str] = {ScopeType.openid, ScopeType.profile}
 
     @classmethod
@@ -247,7 +247,7 @@ def get_userinfo(cls, user: models_core.CoreUser):
 
 class MinecraftAuthClient(BaseAuthClient):
     # Set of scopes the auth client is authorized to grant when issuing an access token.
-    # See app.utils.types.scopes_type.ScopeType for possible values
+    # See app.types.scopes_type.ScopeType for possible values
     allowed_scopes: set[ScopeType | str] = {ScopeType.profile}
 
     @classmethod
@@ -262,7 +262,7 @@ def get_userinfo(cls, user: models_core.CoreUser):
 
 class ChallengerAuthClient(BaseAuthClient):
     # Set of scopes the auth client is authorized to grant when issuing an access token.
-    # See app.utils.types.scopes_type.ScopeType for possible values
+    # See app.types.scopes_type.ScopeType for possible values
     allowed_scopes: set[ScopeType | str] = {ScopeType.openid, ScopeType.profile}
 
     @classmethod
@@ -277,7 +277,7 @@ def get_userinfo(cls, user: models_core.CoreUser):
 
 class OpenProjectAuthClient(BaseAuthClient):
     # Set of scopes the auth client is authorized to grant when issuing an access token.
-    # See app.utils.types.scopes_type.ScopeType for possible values
+    # See app.types.scopes_type.ScopeType for possible values
     allowed_scopes: set[ScopeType | str] = {ScopeType.openid, ScopeType.profile}
 
     @classmethod

migrations/env.py

@@ -63,7 +63,14 @@ def run_migrations_offline() -> None:
 
 
 def do_run_migrations(connection: Connection) -> None:
-    context.configure(connection=connection, target_metadata=target_metadata)
+    context.configure(
+        connection=connection,
+        target_metadata=target_metadata,
+        # We don't want our custom type to be prefixed by the whole module path `app.types.datetime.`
+        # because we don't want to have to import it in the migration file.
+        # See https://alembic.sqlalchemy.org/en/latest/autogenerate.html#controlling-the-module-prefix
+        user_module_prefix="",
+    )
 
     with context.begin_transaction():
         context.run_migrations()

migrations/script.py.mako

@@ -1,21 +1,21 @@
 """${message}
 
-Revision ID: ${up_revision}
-Revises: ${down_revision | comma,n}
 Create Date: ${create_date}
-
 """
-from typing import Sequence, Union
+from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
+from alembic import op
+
+from app.types.sqlalchemy import TZDateTime
+
 ${imports if imports else ""}
 
 # revision identifiers, used by Alembic.
 revision: str = ${repr(up_revision)}
-down_revision: Union[str, None] = ${repr(down_revision)}
-branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
-depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+down_revision: str | None = ${repr(down_revision)}
+branch_labels: str | Sequence[str] | None = ${repr(branch_labels)}
+depends_on: str | Sequence[str] | None = ${repr(depends_on)}
 
 
 def upgrade() -> None:

pyproject.toml

@@ -92,7 +92,6 @@ exclude = '''(?x)(
     | __pycache__
     | .pytest_cache
     | .venv
-    | migration
 )'''
 warn_unreachable = true
 

tests/commons.py

@@ -20,9 +20,9 @@
 from app.core.users import cruds_users
 from app.database import Base
 from app.dependencies import get_db, get_redis_client, get_settings
+from app.types.floors_type import FloorsType
 from app.utils.redis import connect, disconnect
 from app.utils.tools import get_random_string
-from app.utils.types.floors_type import FloorsType
 
 
 @lru_cache

tests/test_auth.py

@@ -7,7 +7,7 @@
 import pytest_asyncio
 
 from app.core import models_core
-from app.utils.types.floors_type import FloorsType
+from app.types.floors_type import FloorsType
 
 # We need to import event_loop for pytest-asyncio routine defined bellow
 from tests.commons import (