CICD improvements (#51)

* cicd: Add dependency.toml files to integrate dependency cascade

* cicd: Isolate Python SDK testing into a separate action

* cicd: Change client tests to use matrix strategy

* chore: Reformat failing file

* fix: CICD typo

* cicd: Change matrix strategy

* fix: Typo in CICD

* cicd: rework all quality jobs to be ran in parallel

* cicd: Isolate test env cleanup into its own action

* cicd: Isolate test env setup into its own action

* chore: Rename python sdk test action

* cicd: Improve documentation for publish jobs

* cicd: Improve docs for quality jobs

* cicd: Remove matrix strategy from CICD tests

.github/actions/cleanup_sdk_test_environment/action.yaml

@@ -0,0 +1,11 @@
+name: 'Cleanup Test Environment'
+description: 'Cleans up the test environment'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Cleanup test environment
+      shell: bash
+      run: |
+        cd client_sdks
+        docker compose -f docker-compose.test.yml down

.github/actions/setup_sdk_test_environment/action.yaml

@@ -0,0 +1,37 @@
+name: 'Setup Test Environment'
+description: 'Sets up the test environment with Docker and database'
+
+inputs:
+  artifact-path:
+    description: 'Path to the Docker image artifact'
+    required: true
+    default: '/tmp'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Download Docker image
+      uses: actions/download-artifact@v4
+      with:
+        name: app-image
+        path: ${{ inputs.artifact-path }}
+
+    - name: Load Docker image
+      shell: bash
+      run: docker load --input ${{ inputs.artifact-path }}/app-image.tar
+
+    - name: Start test environment
+      shell: bash
+      run: |
+        cd client_sdks
+        docker compose -f docker-compose.test.yml up -d --wait
+
+    - name: Migrate database
+      shell: bash
+      run: |
+        cd client_sdks
+        docker compose -f docker-compose.test.yml exec tacoq /bin/bash -c "
+          cargo install sqlx-cli --no-default-features --features postgres &&
+          cargo sqlx database create &&
+          cargo sqlx migrate run --source ./server/libs/db-common/migrations
+        "

.github/actions/test_python_sdk/action.yml

@@ -0,0 +1,28 @@
+name: 'Python SDK Test Steps'
+description: 'Runs tests for the Python SDK'
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-python@v4
+      with:
+        python-version: "3.x"
+
+    - name: Install UV
+      shell: bash
+      run: curl -LsSf https://astral.sh/uv/install.sh | sh
+
+    - name: Install dependencies
+      shell: bash
+      run: |
+        cd client_sdks/python
+        uv sync --dev
+
+    - name: Run tests
+      shell: bash
+      env:
+        MANAGER_TEST_URL: http://localhost:3000
+        BROKER_TEST_URL: amqp://user:password@localhost:5672
+      run: |
+        cd client_sdks/python
+        uv run pytest 

.github/workflows/publish.yml

@@ -6,6 +6,11 @@ on:
     tags: ["v*"]
 
 jobs:
+
+  # =================================================
+  # Publish Manager
+  # =================================================
+
   publish_manager:
     name: Publish Docker image
     runs-on: ubuntu-latest
@@ -47,7 +52,11 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
-  publish_sdk:
+  # =================================================
+  # Publish Python SDK
+  # =================================================
+
+  publish_python_sdk:
     name: Publish Python SDK
     runs-on: ubuntu-latest
     permissions:

.github/workflows/quality.yml

@@ -6,46 +6,62 @@ on:
       - main
 
 jobs:
-  quality:
-    name: Code Quality Checks
+
+  # ================================
+  # Rust Formatting & Linting
+  # ================================
+
+  rust-fmt:
+    name: Rust Formatting
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-
-      # Rust checks
       - uses: dtolnay/rust-toolchain@stable
         with:
-          components: rustfmt, clippy
-
+          components: rustfmt
       - name: Check code formatting
         run: cargo fmt --all -- --check
 
+  rust-clippy:
+    name: Rust Clippy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy
       - name: Check clippy
         run: cargo clippy --all-features --tests -- -D warnings
 
+  rust-sqlx:
+    name: SQLx Schema Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
       - name: Install sqlx-cli
         run: cargo install sqlx-cli --no-default-features --features native-tls,postgres
-
       - name: Run sqlx prepare
         run: cargo sqlx prepare --workspace --check
         env:
           SQLX_OFFLINE: true
 
-      # Python checks
-      - name: Set up Python
-        uses: actions/setup-python@v4
+  # ================================
+  # Python Formatting & Linting
+  # ================================
+
+  python-quality:
+    name: Python Quality
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.12"
-
       - name: Install uv
         run: curl -LsSf https://astral.sh/uv/install.sh | sh
-
-      - name: Check Python formatting
+      - name: Check formatting & linting
         run: |
           cd client_sdks/python
           uv tool run ruff format --check .
-
-      - name: Run Python linter
-        run: |
-          cd client_sdks/python
           uv tool run ruff check .

.github/workflows/test.yml

@@ -6,6 +6,11 @@ on:
       - main
 
 jobs:
+
+  # =================================================
+  # Build & Image Upload
+  # =================================================
+
   build:
     name: Build Docker image
     runs-on: ubuntu-latest
@@ -32,9 +37,15 @@ jobs:
           name: app-image
           path: /tmp/app-image.tar
 
-  test:
-    name: Run tests
-    needs: build
+  # =================================================
+  # Server Tests
+  # These only depend on the database, so we run the
+  # postgres service in the same container.
+  # =================================================
+
+  server-tests:
+    name: Run server tests
+    needs: [build]
     runs-on: ubuntu-latest
     env:
       DATABASE_URL: postgres://postgres:postgres@localhost:5432/test_db
@@ -72,61 +83,30 @@ jobs:
             -e DATABASE_URL=${DATABASE_URL} \
             tacoq-manager cargo test --release --all-features
 
-  e2e-test:
-    name: End-to-end tests
+  # =================================================
+  # SDK Tests
+  # These also act as end-to-end tests for the server
+  # so we need to set up both Postgres AND the server
+  # in the same container.
+  # =================================================
+
+  python-sdk-tests:
+    name: Python SDK Tests
+    needs: [build]
     runs-on: ubuntu-latest
-    needs: [build, test]
 
     steps:
       - uses: actions/checkout@v4
 
-      - name: Download Docker image
-        uses: actions/download-artifact@v4
+      # Setup
+      - uses: ./.github/actions/setup_sdk_test_environment
         with:
-          name: app-image
-          path: /tmp
-
-      - name: Load Docker image
-        run: docker load --input /tmp/app-image.tar
+          artifact-path: /tmp
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.x"
+      # Python Tests
+      - name: Run Python tests
+        uses: ./.github/actions/test_python_sdk
 
-      - name: Install uv
-        run: |
-          curl -LsSf https://astral.sh/uv/install.sh | sh
-
-      - name: Install Python dependencies
-        run: |
-          cd client_sdks/python
-          uv sync --dev
-
-      - name: Start test environment
-        run: |
-          cd client_sdks
-          docker compose -f docker-compose.test.yml up -d --wait
-
-      - name: Migrate database
-        run: |
-          cd client_sdks
-          docker compose -f docker-compose.test.yml exec tacoq /bin/bash -c "
-            cargo install sqlx-cli --no-default-features --features postgres &&
-            cargo sqlx database create &&
-            cargo sqlx migrate run --source ./server/libs/db-common/migrations
-          "
-
-      - name: Run end-to-end tests
-        env:
-          MANAGER_TEST_URL: http://localhost:3000
-          BROKER_TEST_URL: amqp://user:password@localhost:5672
-        run: |
-          cd client_sdks/python
-          uv run pytest
-
-      - name: Cleanup test environment
+      # Cleanup
+      - uses: ./.github/actions/cleanup_sdk_test_environment
         if: always()
-        run: |
-          cd client_sdks
-          docker compose -f docker-compose.test.yml down

client_sdks/dependencies.toml

@@ -0,0 +1,13 @@
+[module]
+name = "clients"
+
+[dependencies]
+manager = { name = "manager" }
+
+[file_paths]
+include = [
+  "python/**/*",
+  "rust/**/*",
+  "typescript/**/*",
+  "docker-compose.test.yml",
+]

client_sdks/python/tests/manager/test_health_check.py

@@ -8,6 +8,6 @@ async def test_health_check_client(manager_client: ManagerClient):
 
     health_state = await manager_client.check_health()
 
-    assert (
-        health_state == ManagerStates.HEALTHY
-    ), f"Manager is not healthy. Current state: {health_state}"
+    assert health_state == ManagerStates.HEALTHY, (
+        f"Manager is not healthy. Current state: {health_state}"
+    )

server/dependencies.toml

@@ -0,0 +1,8 @@
+[module]
+name = "server"
+
+[file_paths]
+include = [
+  "services/**/*",
+  "libs/**/*",
+]