Merge pull request #107 from piercefreeman/feature/database-migration…

…s-v1

Add V1 of database schema migrator

Makefile

@@ -95,7 +95,11 @@ lint-validation-scripts:
 
 # Tests
 test-lib:
+	(cd $(LIB_DIR) && docker-compose -f docker-compose.test.yml up -d)
+	@$(call wait-for-postgres,30,5438)
+	@set -e; \
 	$(call test-common,$(LIB_DIR),$(LIB_NAME))
+	(cd $(LIB_DIR) && docker-compose -f docker-compose.test.yml down)
 	$(call test-rust-common,$(LIB_DIR),$(LIB_NAME))
 test-create-mountaineer-app:
 	$(call test-common,$(CREATE_MOUNTAINEER_APP_DIR),$(CREATE_MOUNTAINEER_APP_NAME))

docker-compose.test.yml

@@ -0,0 +1,16 @@
+version: "3.8"
+
+services:
+  postgres:
+    image: postgres:latest
+    environment:
+      POSTGRES_USER: mountaineer
+      POSTGRES_PASSWORD: mysecretpassword
+      POSTGRES_DB: mountaineer_test_db
+    ports:
+      - "5438:5432"
+    volumes:
+      - postgres_data_test:/var/lib/postgresql/data
+
+volumes:
+  postgres_data_test:

mountaineer/__tests__/migrations/conftest.py

@@ -0,0 +1,135 @@
+from contextlib import asynccontextmanager, contextmanager
+from os import environ
+from warnings import filterwarnings
+
+import pytest
+import pytest_asyncio
+from fastapi import Depends
+from sqlalchemy import exc as sa_exc
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
+from sqlmodel import SQLModel, text
+
+from mountaineer.config import ConfigBase, unregister_config
+from mountaineer.database import DatabaseDependencies
+from mountaineer.database.config import DatabaseConfig
+from mountaineer.dependencies.base import get_function_dependencies
+from mountaineer.test_utilities import bootstrap_database
+
+
+@contextmanager
+def clear_registration_metadata():
+    """
+    Temporarily clear the sqlalchemy metadata
+
+    """
+    archived_tables = SQLModel.metadata.tables
+    archived_schemas = SQLModel.metadata._schemas
+    archived_memos = SQLModel.metadata._fk_memos
+
+    try:
+        SQLModel.metadata.clear()
+        yield
+    finally:
+        # Restore
+        SQLModel.metadata.tables = archived_tables
+        SQLModel.metadata._schemas = archived_schemas
+        SQLModel.metadata._fk_memos = archived_memos
+
+
+@pytest_asyncio.fixture
+async def clear_all_database_objects(db_session: AsyncSession):
+    """
+    Clear all database objects, including those not directly created through
+    SQLAlchemy.
+
+    """
+    # Step 1: Drop all tables in the public schema
+    await db_session.execute(
+        text(
+            """
+        DO $$ DECLARE
+            r RECORD;
+        BEGIN
+            FOR r IN (SELECT tablename FROM pg_tables WHERE schemaname = 'public') LOOP
+                EXECUTE 'DROP TABLE IF EXISTS ' || quote_ident(r.tablename) || ' CASCADE';
+            END LOOP;
+        END $$;
+    """
+        )
+    )
+
+    # Step 2: Drop all custom types in the public schema
+    await db_session.execute(
+        text(
+            """
+        DO $$ DECLARE
+            r RECORD;
+        BEGIN
+            FOR r IN (SELECT typname FROM pg_type WHERE typtype = 'e' AND typnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'public')) LOOP
+                EXECUTE 'DROP TYPE IF EXISTS ' || quote_ident(r.typname) || ' CASCADE';
+            END LOOP;
+        END $$;
+    """
+        )
+    )
+
+    await db_session.commit()
+
+
+@pytest.fixture
+def isolated_sqlalchemy(clear_all_database_objects):
+    """
+    Drops database tables and clears the metadata that is registered
+    in-memory, just for this test
+
+    """
+    # Avoid also creating the tables for other SQLModels that have been defined
+    # in memory (and therefore captured in the same registry)
+    with clear_registration_metadata():
+        # Overrides the warning that we see when creating multiple ExampleDBModels
+        # in one session
+        filterwarnings("ignore", category=sa_exc.SAWarning)
+
+        yield
+
+
+class MigrationAppConfig(ConfigBase, DatabaseConfig):
+    pass
+
+
+@pytest.fixture(autouse=True)
+def config():
+    """
+    Test-time configuration. Set to auto-use the fixture so that the configuration
+    is mounted and exposed to the dependency injection framework in all tests.
+
+    """
+    unregister_config()
+    return MigrationAppConfig(
+        POSTGRES_HOST=environ.get("TEST_POSTGRES_HOST", "localhost"),
+        POSTGRES_USER=environ.get("TEST_POSTGRES_USER", "mountaineer"),
+        POSTGRES_PASSWORD=environ.get("TEST_POSTGRES_PASSWORD", "mysecretpassword"),
+        POSTGRES_DB=environ.get("TEST_POSTGRES_DB", "mountaineer_test_db"),
+        POSTGRES_PORT=int(environ.get("POSTGRES_PORT", "5438")),
+    )
+
+
+@pytest_asyncio.fixture(scope="function")
+async def db_engine(config: MigrationAppConfig):
+    @asynccontextmanager
+    async def run_bootstrap(
+        engine: AsyncEngine = Depends(DatabaseDependencies.get_db),
+    ):
+        await bootstrap_database(engine)
+        yield engine
+
+    async with get_function_dependencies(callable=run_bootstrap) as values:
+        async with run_bootstrap(**values) as engine:
+            yield engine
+
+
+@pytest_asyncio.fixture
+async def db_session(db_engine: AsyncEngine):
+    session_maker = async_sessionmaker(db_engine, expire_on_commit=False)
+    async with session_maker() as session:
+        yield session

mountaineer/__tests__/migrations/test_actions.py

@@ -0,0 +1,50 @@
+from unittest.mock import AsyncMock
+
+import pytest
+
+from mountaineer.migrations.actions import DatabaseActions, DryRunAction
+
+
+def example_action_fn(arg_1: str):
+    pass
+
+
+@pytest.mark.asyncio
+async def test_record_signature_dry_run():
+    database_actions = DatabaseActions(dry_run=True)
+
+    await database_actions._record_signature(
+        example_action_fn, {"arg_1": "test"}, "SQL"
+    )
+
+    assert database_actions.dry_run_actions == [
+        DryRunAction(fn=example_action_fn, kwargs={"arg_1": "test"})
+    ]
+    assert database_actions.prod_sqls == []
+
+
+@pytest.mark.asyncio
+async def test_record_signature_prod():
+    database_actions = DatabaseActions(dry_run=False, db_session=AsyncMock())
+
+    await database_actions._record_signature(
+        example_action_fn, {"arg_1": "test"}, "SQL"
+    )
+
+    assert database_actions.dry_run_actions == []
+    assert database_actions.prod_sqls == ["SQL"]
+
+
+@pytest.mark.asyncio
+async def test_record_signature_incorrect_kwarg():
+    database_actions = DatabaseActions(dry_run=False, db_session=AsyncMock())
+
+    # An extra, non-existent kwarg is provided
+    with pytest.raises(ValueError):
+        await database_actions._record_signature(
+            example_action_fn, {"arg_1": "test", "arg_2": "test"}, "SQL"
+        )
+
+    # A required kwarg is missing
+    with pytest.raises(ValueError):
+        await database_actions._record_signature(example_action_fn, {}, "SQL")