✨Meet the clusters-keeper service (⚠️ devops)

.github/CODEOWNERS

@@ -26,6 +26,7 @@ Makefile                                  @pcrespov @sanderegg
 /services/api-server/                     @pcrespov
 /services/autoscaling/                    @sanderegg @pcrespov
 /services/catalog/                        @pcrespov @sanderegg
+/services/clusters-keeper/                @sanderegg
 /services/datcore-adapter/                @sanderegg
 /services/director*/                      @sanderegg @pcrespov @GitHK
 /services/docker-compose*.yml             @sanderegg @mrnicegyu11 @YuryHrytsuk

.github/workflows/ci-testing-deploy.yml

@@ -62,6 +62,7 @@ jobs:
       api-server: ${{ steps.filter.outputs.api-server }}
       autoscaling: ${{ steps.filter.outputs.autoscaling }}
       catalog: ${{ steps.filter.outputs.catalog }}
+      clusters-keeper: ${{ steps.filter.outputs.clusters-keeper }}
       dask-sidecar: ${{ steps.filter.outputs.dask-sidecar }}
       datcore-adapter: ${{ steps.filter.outputs.datcore-adapter }}
       director: ${{ steps.filter.outputs.director }}
@@ -147,6 +148,12 @@ jobs:
               - 'services/docker-compose*'
               - 'scripts/mypy/*'
               - 'mypy.ini'
+            clusters-keeper:
+              - 'packages/**'
+              - 'services/clusters-keeper/**'
+              - 'services/docker-compose*'
+              - 'scripts/mypy/*'
+              - 'mypy.ini'
             dask-sidecar:
               - 'packages/**'
               - 'services/dask-sidecar/**'
@@ -666,6 +673,61 @@ jobs:
         with:
           flags: unittests #optional
 
+  unit-test-clusters-keeper:
+    needs: changes
+    if: ${{ needs.changes.outputs.clusters-keeper == 'true' || github.event_name == 'push' }}
+    timeout-minutes: 18 # if this timeout gets too small, then split the tests
+    name: "[unit] clusters-keeper"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: ["3.10"]
+        os: [ubuntu-22.04]
+        docker_buildx: [v0.10.4]
+        docker_compose: [1.29.1]
+        include:
+          - docker_compose: 1.29.1
+            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v3
+      - name: setup docker buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          version: ${{ matrix.docker_buildx }}
+          driver: docker-container
+      - name: setup docker-compose
+        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
+      - name: setup python environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+          cache: "pip"
+          cache-dependency-path: "services/clusters-keeper/requirements/ci.txt"
+      - name: show system version
+        run: ./ci/helpers/show_system_versions.bash
+      - name: install
+        run: |
+          make devenv
+          source .venv/bin/activate && \
+          pushd services/clusters-keeper && \
+          make install-ci
+      - name: typecheck
+        run: |
+          source .venv/bin/activate && \
+          pushd services/clusters-keeper && \
+          make mypy
+      - name: test
+        if: always()
+        run: |
+          source .venv/bin/activate && \
+          pushd services/clusters-keeper && \
+          make test-ci-unit
+      - uses: codecov/[email protected]
+        with:
+          flags: unittests #optional
+
   unit-test-datcore-adapter:
     needs: changes
     if: ${{ needs.changes.outputs.datcore-adapter == 'true' || github.event_name == 'push' }}
@@ -1458,6 +1520,7 @@ jobs:
         unit-test-api-server,
         unit-test-autoscaling,
         unit-test-catalog,
+        unit-test-clusters-keeper,
         unit-test-dask-sidecar,
         unit-test-dask-task-models-library,
         unit-test-datcore-adapter,

.ruff.toml

@@ -1,73 +1,73 @@
 
 select = [
-    "A", # [https://pypi.org/project/flake8-builtins/]
-    "ARG", # [https://pypi.org/project/flake8-unused-arguments/]
-    "ASYNC", # [https://pypi.org/project/flake8-async/]
-    "B",  # [https://pypi.org/project/flake8-bugbear/]
-    "C4",  # [https://pypi.org/project/flake8-comprehensions/]
-    "C90", # [https://pypi.org/project/mccabe/] (complexity)
-    "DTZ", # [https://pypi.org/project/flake8-datetimez/]
-    "E",  # [https://pypi.org/project/pycodestyle/] errors
-    "EM", # [https://pypi.org/project/flake8-errmsg/]
-    "ERA", # [https://pypi.org/project/eradicate/]
-    "F",  # [https://pypi.org/project/pyflakes/]
-    "FBT", # [https://pypi.org/project/flake8-boolean-trap/]
-    "FIX", # [https://github.com/tommilligan/flake8-fixme]
-    "G", # [https://pypi.org/project/flake8-logging-format/0.9.0/]
-    "I",  # [https://pypi.org/project/isort/]
-    "ICN", # [https://github.com/joaopalmeiro/flake8-import-conventions]
-    "ISC", # [https://pypi.org/project/flake8-implicit-str-concat/]
-    "N", # [https://pypi.org/project/pep8-naming/]
-    "NPY", # NumPy-speficic rules
-    "PERF", # [https://pypi.org/project/perflint/]
-    "PIE", # [https://pypi.org/project/flake8-pie/]
-    "PL", # [https://pypi.org/project/pylint/]
-    "PT", # [https://pypi.org/project/flake8-pytest-style/]
-    "PTH", # [https://pypi.org/project/flake8-use-pathlib/]
-    "PYI", # [https://pypi.org/project/flake8-pyi/]
-    "RET", # [https://pypi.org/project/flake8-return/]
-    "RSE", # [https://pypi.org/project/flake8-raise/]
-    "RUF", # RUFF-specific rules
-    "S", # [https://pypi.org/project/flake8-bandit/] (Automated security testing)
-    "SIM", # [https://pypi.org/project/flake8-simplify/]
-    "SLF", # [https://pypi.org/project/flake8-self/]
-    "SLOT", # [https://pypi.org/project/flake8-slots/]
-    "T10", # https://pypi.org/project/flake8-debugger/
-    "T20", # [https://pypi.org/project/flake8-print/]
-    "TID", # [https://pypi.org/project/flake8-tidy-imports/]
-    "TRY", # [https://pypi.org/project/tryceratops/1.1.0/] (exception anti-patterns)
-    "UP", # [https://pypi.org/project/pyupgrade/]
-    "W",  # [https://pypi.org/project/pycodestyle/] warnings
-    "YTT", # [https://pypi.org/project/flake8-2020/]
+  "A",     # [https://pypi.org/project/flake8-builtins/]
+  "ARG",   # [https://pypi.org/project/flake8-unused-arguments/]
+  "ASYNC", # [https://pypi.org/project/flake8-async/]
+  "B",     # [https://pypi.org/project/flake8-bugbear/]
+  "C4",    # [https://pypi.org/project/flake8-comprehensions/]
+  "C90",   # [https://pypi.org/project/mccabe/] (complexity)
+  "DTZ",   # [https://pypi.org/project/flake8-datetimez/]
+  "E",     # [https://pypi.org/project/pycodestyle/] errors
+  "EM",    # [https://pypi.org/project/flake8-errmsg/]
+  "ERA",   # [https://pypi.org/project/eradicate/]
+  "F",     # [https://pypi.org/project/pyflakes/]
+  "FBT",   # [https://pypi.org/project/flake8-boolean-trap/]
+  "FIX",   # [https://github.com/tommilligan/flake8-fixme]
+  "G",     # [https://pypi.org/project/flake8-logging-format/0.9.0/]
+  "I",     # [https://pypi.org/project/isort/]
+  "ICN",   # [https://github.com/joaopalmeiro/flake8-import-conventions]
+  "ISC",   # [https://pypi.org/project/flake8-implicit-str-concat/]
+  "N",     # [https://pypi.org/project/pep8-naming/]
+  "NPY",   # NumPy-speficic rules
+  "PERF",  # [https://pypi.org/project/perflint/]
+  "PIE",   # [https://pypi.org/project/flake8-pie/]
+  "PL",    # [https://pypi.org/project/pylint/]
+  "PT",    # [https://pypi.org/project/flake8-pytest-style/]
+  "PTH",   # [https://pypi.org/project/flake8-use-pathlib/]
+  "PYI",   # [https://pypi.org/project/flake8-pyi/]
+  "RET",   # [https://pypi.org/project/flake8-return/]
+  "RSE",   # [https://pypi.org/project/flake8-raise/]
+  "RUF",   # RUFF-specific rules
+  "S",     # [https://pypi.org/project/flake8-bandit/] (Automated security testing)
+  "SIM",   # [https://pypi.org/project/flake8-simplify/]
+  "SLF",   # [https://pypi.org/project/flake8-self/]
+  "SLOT",  # [https://pypi.org/project/flake8-slots/]
+  "T10",   # https://pypi.org/project/flake8-debugger/
+  "T20",   # [https://pypi.org/project/flake8-print/]
+  "TID",   # [https://pypi.org/project/flake8-tidy-imports/]
+  "TRY",   # [https://pypi.org/project/tryceratops/1.1.0/] (exception anti-patterns)
+  "UP",    # [https://pypi.org/project/pyupgrade/]
+  "W",     # [https://pypi.org/project/pycodestyle/] warnings
+  "YTT",   # [https://pypi.org/project/flake8-2020/]
 ]
 ignore = [
-    "E501",  # line too long, handled by black
-    "S101", # use of `assert` detected hanbled by pylance, does not support noseq
-    "TID252", # [*] Relative imports from parent modules are banned
-    "TRY300", # Checks for return statements in try blocks. SEE https://beta.ruff.rs/docs/rules/try-consider-else/
+  "E501",   # line too long, handled by black
+  "S101",   # use of `assert` detected hanbled by pylance, does not support noseq
+  "TID252", # [*] Relative imports from parent modules are banned
+  "TRY300", # Checks for return statements in try blocks. SEE https://beta.ruff.rs/docs/rules/try-consider-else/
 ]
 
 target-version = "py310"
 
 
 [per-file-ignores]
 "**/tests/**" = [
-  "T201", # print found
-  "ARG001", # unused function argument
-  "PT019", # user pytest.mark.usefixture
+  "T201",    # print found
+  "ARG001",  # unused function argument
+  "PT019",   # user pytest.mark.usefixture
   "PLR2004", # use of magic values
   "PLR0913", # too many arguments
-  "N806", # Uppercase variables in functions
-  "PT001", # use pytest.fixture over pytest.fixture() whatsoever
-  "ERA001", # found commented out code
-  "FBT001", # Boolean positional arg in function definition
+  "N806",    # Uppercase variables in functions
+  "PT001",   # use pytest.fixture over pytest.fixture() whatsoever
+  "PT004",   # does not return anythin, add leading underscore
+  "ERA001",  # found commented out code
+  "FBT001",  # Boolean positional arg in function definition
 ]
 
 [flake8-pytest-style]
 fixture-parentheses = false
 parametrize-names-type = "csv"
 
 
-
 [pylint]
 max-args = 10

Makefile

@@ -35,6 +35,7 @@ SERVICES_NAMES_TO_BUILD := \
   api-server \
   autoscaling \
   catalog \
+	clusters-keeper \
   dask-sidecar \
   datcore-adapter \
   director \

packages/pytest-simcore/src/pytest_simcore/simcore_services.py

@@ -40,6 +40,7 @@
 DEFAULT_SERVICE_HEALTHCHECK_ENTRYPOINT = "/v0/"
 MAP_SERVICE_HEALTHCHECK_ENTRYPOINT = {
     "autoscaling": "/",
+    "clusters-keeper": "/",
     "dask-scheduler": "/health",
     "datcore-adapter": "/v0/live",
     "director-v2": "/",