✨adding dynamic-scheduler service (⚠️ devops)

.github/workflows/ci-testing-deploy.yml

@@ -72,6 +72,7 @@ jobs:
       migration: ${{ steps.filter.outputs.migration }}
       osparc-gateway-server: ${{ steps.filter.outputs.osparc-gateway-server }}
       payments: ${{ steps.filter.outputs.payments }}
+      dynamic-scheduler: ${{ steps.filter.outputs.dynamic-scheduler }}
       resource-usage-tracker: ${{ steps.filter.outputs.resource-usage-tracker }}
       static-webserver: ${{ steps.filter.outputs.static-webserver }}
       storage: ${{ steps.filter.outputs.storage }}
@@ -205,6 +206,12 @@ jobs:
                 - 'services/docker-compose*'
                 - 'scripts/mypy/*'
                 - 'mypy.ini'
+            dynamic-scheduler:
+                - 'packages/**'
+                - 'services/dynamic-scheduler/**'
+                - 'services/docker-compose*'
+                - 'scripts/mypy/*'
+                - 'mypy.ini'
             resource-usage-tracker:
               - 'packages/**'
               - 'services/resource-usage-tracker/**'
@@ -963,6 +970,45 @@ jobs:
         with:
           flags: unittests #optional
 
+  unit-test-dynamic-scheduler:
+    needs: changes
+    if: ${{ needs.changes.outputs.dynamic-scheduler == 'true' || github.event_name == 'push' }}
+    timeout-minutes: 18 # if this timeout gets too small, then split the tests
+    name: "[unit] dynamic-scheduler"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: ["3.10"]
+        os: [ubuntu-22.04]
+        docker_buildx: [v0.10.4]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+      - name: setup docker buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: ${{ matrix.docker_buildx }}
+          driver: docker-container
+      - name: setup python environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+          cache: "pip"
+          cache-dependency-path: "services/dynamic-scheduler/requirements/ci.txt"
+      - name: show system version
+        run: ./ci/helpers/show_system_versions.bash
+      - name: install
+        run: ./ci/github/unit-testing/dynamic-scheduler.bash install
+      - name: typecheck
+        run: ./ci/github/unit-testing/dynamic-scheduler.bash typecheck
+      - name: test
+        if: always()
+        run: ./ci/github/unit-testing/dynamic-scheduler.bash test
+      - uses: codecov/[email protected]
+        with:
+          flags: unittests #optional
+
   unit-test-resource-usage-tracker:
     needs: changes
     if: ${{ needs.changes.outputs.resource-usage-tracker == 'true' || github.event_name == 'push' }}
@@ -1410,6 +1456,7 @@ jobs:
         unit-test-models-library,
         unit-test-osparc-gateway-server,
         unit-test-payments,
+        unit-test-dynamic-scheduler,
         unit-test-postgres-database,
         unit-test-python-linting,
         unit-test-resource-usage-tracker,

Makefile

@@ -46,6 +46,7 @@ SERVICES_NAMES_TO_BUILD := \
 	osparc-gateway-server \
 	payments \
 	resource-usage-tracker \
+	dynamic-scheduler \
   service-integration \
   static-webserver \
   storage \
@@ -70,6 +71,7 @@ export DIRECTOR_V2_API_VERSION:= $(shell cat $(CURDIR)/services/director-v2/VERS
 export STORAGE_API_VERSION    := $(shell cat $(CURDIR)/services/storage/VERSION)
 export INVITATIONS_API_VERSION  := $(shell cat $(CURDIR)/services/invitations/VERSION)
 export PAYMENTS_API_VERSION  := $(shell cat $(CURDIR)/services/payments/VERSION)
+export DYNAMIC_SCHEDULER_API_VERSION  := $(shell cat $(CURDIR)/services/dynamic-scheduler/VERSION)
 export DATCORE_ADAPTER_API_VERSION    := $(shell cat $(CURDIR)/services/datcore-adapter/VERSION)
 export WEBSERVER_API_VERSION  := $(shell cat $(CURDIR)/services/web/server/VERSION)
 
@@ -571,6 +573,7 @@ settings-schema.json: ## [container] dumps json-schema settings of all services
 	@$(MAKE_C) services/director-v2 $@
 	@$(MAKE_C) services/invitations $@
 	@$(MAKE_C) services/payments $@
+	@$(MAKE_C) services/dynamic-scheduler $@
 	@$(MAKE_C) services/storage $@
 	@$(MAKE_C) services/web/server $@
 

ci/github/unit-testing/dynamic-scheduler.bash

@@ -0,0 +1,41 @@
+#!/bin/bash
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -o errexit  # abort on nonzero exitstatus
+set -o nounset  # abort on unbound variable
+set -o pipefail # don't hide errors within pipes
+IFS=$'\n\t'
+
+install() {
+  bash ci/helpers/ensure_python_pip.bash
+  make devenv
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  pushd services/dynamic-scheduler
+  make install-ci
+  popd
+  .venv/bin/pip list --verbose
+}
+
+test() {
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  pushd services/dynamic-scheduler
+  make test-ci-unit
+  popd
+}
+
+typecheck() {
+  pushd services/dynamic-scheduler
+  make mypy
+  popd
+}
+
+# Check if the function exists (bash specific)
+if declare -f "$1" >/dev/null; then
+  # call arguments verbatim
+  "$@"
+else
+  # Show a helpful error
+  echo "'$1' is not a known function name" >&2
+  exit 1
+fi

packages/models-library/src/models_library/api_schemas_catalog/meta.py → packages/models-library/src/models_library/api_schemas__common/meta.py

@@ -5,11 +5,11 @@
 from ..basic_types import VersionStr
 
 
-class Meta(BaseModel):
+class BaseMeta(BaseModel):
     name: str
     version: VersionStr
     released: dict[str, VersionStr] | None = Field(
-        None, description="Maps every route's path tag with a released version"
+        default=None, description="Maps every route's path tag with a released version"
     )
 
     class Config:

packages/models-library/src/models_library/api_schemas_dynamic_scheduler/__init__.py

@@ -0,0 +1,9 @@
+from typing import Final
+
+from pydantic import parse_obj_as
+
+from ..rabbitmq_basic_types import RPCNamespace
+
+DYNAMIC_SCHEDULER_RPC_NAMESPACE: Final[RPCNamespace] = parse_obj_as(
+    RPCNamespace, "dynamic-scheduler"
+)

packages/pytest-simcore/src/pytest_simcore/simcore_services.py

@@ -45,6 +45,7 @@
     "dask-scheduler": "/health",
     "datcore-adapter": "/v0/live",
     "director-v2": "/",
+    "dynamic-scheduler": "/",
     "invitations": "/",
     "payments": "/",
     "resource-usage-tracker": "/",

packages/settings-library/src/settings_library/director_v2.py

@@ -0,0 +1,30 @@
+from functools import cached_property
+
+from pydantic import parse_obj_as
+from settings_library.base import BaseCustomSettings
+from settings_library.basic_types import PortInt, VersionTag
+from settings_library.utils_service import (
+    DEFAULT_FASTAPI_PORT,
+    MixinServiceSettings,
+    URLPart,
+)
+
+
+class DirectorV2Settings(BaseCustomSettings, MixinServiceSettings):
+    DIRECTOR_V2_HOST: str = "director-v2"
+    DIRECTOR_V2_PORT: PortInt = DEFAULT_FASTAPI_PORT
+    DIRECTOR_V2_VTAG: VersionTag = parse_obj_as(VersionTag, "v1")
+
+    @cached_property
+    def api_base_url(self) -> str:
+        # http://director-v2:8000/v1
+        return self._compose_url(
+            prefix="DIRECTOR_V2", port=URLPart.REQUIRED, vtag=URLPart.REQUIRED
+        )
+
+    @cached_property
+    def base_url(self) -> str:
+        # http://director-v2:8000
+        return self._compose_url(
+            prefix="DIRECTOR_V2", port=URLPart.REQUIRED, vtag=URLPart.EXCLUDE
+        )

services/api-server/src/simcore_service_api_server/api/routes/meta.py

@@ -16,8 +16,8 @@ async def get_service_metadata(
 ):
     return Meta(
         name=__name__.split(".")[0],
-        version=API_VERSION,  # type: ignore
-        released={API_VTAG: API_VERSION},  # type: ignore
+        version=API_VERSION,
+        released={API_VTAG: API_VERSION},
         docs_url=url_for("swagger_ui_html"),
         docs_dev_url=url_for("swagger_ui_html"),
     )

services/api-server/src/simcore_service_api_server/models/schemas/meta.py

@@ -1,19 +1,15 @@
-from pydantic import AnyHttpUrl, BaseModel, Field
+from typing import ClassVar
 
-from ..basic_types import VersionStr
+from models_library.api_schemas__common.meta import BaseMeta
+from pydantic import AnyHttpUrl
 
 
-class Meta(BaseModel):
-    name: str
-    version: VersionStr
-    released: dict[str, VersionStr] | None = Field(
-        None, description="Maps every route's path tag with a released version"
-    )
+class Meta(BaseMeta):
     docs_url: AnyHttpUrl
     docs_dev_url: AnyHttpUrl
 
     class Config:
-        schema_extra = {
+        schema_extra: ClassVar = {
             "example": {
                 "name": "simcore_service_foo",
                 "version": "2.4.45",

services/api-server/tests/unit/test_api_meta.py

@@ -4,8 +4,8 @@
 
 import pytest
 from httpx import AsyncClient
+from models_library.api_schemas__common.meta import BaseMeta
 from simcore_service_api_server._meta import API_VERSION, API_VTAG
-from simcore_service_api_server.models.schemas.meta import Meta
 
 pytestmark = pytest.mark.asyncio
 
@@ -16,5 +16,5 @@ async def test_read_service_meta(client: AsyncClient):
     assert response.status_code == 200
     assert response.json()["version"] == API_VERSION
 
-    meta = Meta(**response.json())
+    meta = BaseMeta(**response.json())
     assert meta.version == API_VERSION

services/catalog/src/simcore_service_catalog/api/routes/meta.py

@@ -1,15 +1,16 @@
 from fastapi import APIRouter
-from models_library.api_schemas_catalog.meta import Meta, VersionStr
+from models_library.api_schemas__common.meta import BaseMeta
+from models_library.basic_types import VersionStr
 from pydantic import parse_obj_as
 
 from ..._meta import API_VERSION, API_VTAG
 
 router = APIRouter()
 
 
-@router.get("", response_model=Meta)
+@router.get("", response_model=BaseMeta)
 async def get_service_metadata():
-    return Meta(
+    return BaseMeta(
         name=__name__.split(".")[0],
         version=parse_obj_as(VersionStr, API_VERSION),
         released={API_VTAG: parse_obj_as(VersionStr, API_VERSION)},

services/clusters-keeper/Makefile

@@ -21,7 +21,7 @@ PHONY: .init-swarm
 	$(if $(SWARM_HOSTS),,docker swarm init --advertise-addr=$(get_my_ip))
 
 
-.PHONY: test-dask-schduler-deploy
+.PHONY: test-dask-scheduler-deploy
 deploy-dask-stack: .init-swarm ## deploy the dask stack for local testing
 	# using local/dask-sidecar:production images
 	@DOCKER_IMAGE_TAG=production \

services/director-v2/src/simcore_service_director_v2/api/routes/meta.py

@@ -1,14 +1,15 @@
 from fastapi import APIRouter
-from models_library.api_schemas_directorv2.meta import Meta, VersionStr
+from models_library.api_schemas__common.meta import BaseMeta
+from models_library.basic_types import VersionStr
 
 from ...meta import API_VERSION, API_VTAG
 
 router = APIRouter()
 
 
-@router.get("", response_model=Meta)
-async def get_service_metadata() -> Meta:
-    return Meta(
+@router.get("", response_model=BaseMeta)
+async def get_service_metadata() -> BaseMeta:
+    return BaseMeta(
         name=__name__.split(".")[0],
         version=VersionStr(API_VERSION),
         released={API_VTAG: VersionStr(API_VERSION), "v0": VersionStr("0.1.0")},

services/docker-compose-build.yml

@@ -300,6 +300,22 @@ services:
         org.opencontainers.image.revision: "${VCS_REF}"
         io.osparc.api-version: "${PAYMENTS_API_VERSION}"
 
+  dynamic-scheduler:
+    image: local/dynamic-scheduler:${BUILD_TARGET:?build_target_required}
+    build:
+      context: ../
+      dockerfile: services/dynamic-scheduler/Dockerfile
+      cache_from:
+        - local/dynamic-scheduler:${BUILD_TARGET:?build_target_required}
+        - ${DOCKER_REGISTRY:-itisfoundation}/dynamic-scheduler:master-github-latest
+        - ${DOCKER_REGISTRY:-itisfoundation}/dynamic-scheduler:staging-github-latest
+        - ${DOCKER_REGISTRY:-itisfoundation}/dynamic-scheduler:release-github-latest
+      target: ${BUILD_TARGET:?build_target_required}
+      labels:
+        org.opencontainers.image.created: "${BUILD_DATE}"
+        org.opencontainers.image.source: "${VCS_URL}"
+        org.opencontainers.image.revision: "${VCS_REF}"
+        io.osparc.api-version: "${DYNAMIC_SCHEDULER_API_VERSION}"
 
   datcore-adapter:
     image: local/datcore-adapter:${BUILD_TARGET:?build_target_required}

services/docker-compose-deploy.yml

@@ -28,6 +28,8 @@ services:
     image: ${DOCKER_REGISTRY:-itisfoundation}/osparc-gateway-server:${DOCKER_IMAGE_TAG:-latest}
   payments:
     image: ${DOCKER_REGISTRY:-itisfoundation}/payments:${DOCKER_IMAGE_TAG:-latest}
+  dynamic-scheduler:
+    image: ${DOCKER_REGISTRY:-itisfoundation}/dynamic-scheduler:${DOCKER_IMAGE_TAG:-latest}
   resource-usage-tracker:
     image: ${DOCKER_REGISTRY:-itisfoundation}/resource-usage-tracker:${DOCKER_IMAGE_TAG:-latest}
   service-integration:

services/docker-compose.devel.yml

@@ -42,6 +42,15 @@ services:
       - ./payments:/devel/services/payments
       - ../packages:/devel/packages
 
+  dynamic-scheduler:
+    environment:
+      - SC_BOOT_MODE=debug-ptvsd
+      - DYNAMIC_SCHEDULER_LOGLEVEL=DEBUG
+      - DEBUG=true
+    volumes:
+      - ./dynamic-scheduler:/devel/services/dynamic-scheduler
+      - ../packages:/devel/packages
+
   catalog:
     environment:
       - SC_BOOT_MODE=debug-ptvsd

services/docker-compose.local.yml

@@ -67,6 +67,10 @@ services:
     ports:
       - "8011:8000"
 
+  dynamic-scheduler:
+    ports:
+      - "8012:8000"
+
   resource-usage-tracker:
     ports:
       - "8009:8000"