✨ dynamic-sidecar saves and restores user services preferences

packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_service.py

@@ -4,11 +4,10 @@
 
 from pydantic import BaseModel, Field
 
-from ..basic_regex import VERSION_RE
 from ..basic_types import PortInt
 from ..projects import ProjectID
 from ..projects_nodes_io import NodeID
-from ..services import DynamicServiceKey
+from ..services import DynamicServiceKey, ServiceVersion
 from ..services_enums import ServiceBootType, ServiceState
 from ..users import UserID
 
@@ -22,10 +21,9 @@ class CommonServiceDetails(BaseModel):
         ],
         alias="service_key",
     )
-    version: str = Field(
+    version: ServiceVersion = Field(
         ...,
         description="semantic version number of the node",
-        regex=VERSION_RE,
         examples=["1.0.0", "0.0.1"],
         alias="service_version",
     )

packages/models-library/src/models_library/service_settings_labels.py

@@ -289,6 +289,15 @@ class DynamicSidecarServiceLabels(BaseModel):
         ),
     )
 
+    user_preferences_path: Json[Path] | None = Field(
+        None,
+        alias="simcore.service.user-preferences-path",
+        description=(
+            "path where the user user preferences folder "
+            "will be mounted in the user services"
+        ),
+    )
+
     restart_policy: RestartPolicy = Field(
         RestartPolicy.NO_RESTART,
         alias="simcore.service.restart-policy",
@@ -504,6 +513,9 @@ class Config(_BaseConfig):
                             }
                         }
                     ),
+                    "simcore.service.user-preferences-path": json.dumps(
+                        "/tmp/path_to_preferences"  # noqa: S108
+                    ),
                 },
                 # dynamic-service with compose spec
                 {

packages/models-library/src/models_library/user_preferences.py

@@ -1,5 +1,4 @@
 from enum import auto
-from pathlib import Path
 from typing import Annotated, Any, ClassVar, TypeAlias
 
 from pydantic import BaseModel, Field
@@ -13,20 +12,20 @@
 
 
 class _AutoRegisterMeta(ModelMetaclass):
-    _registered_user_preference_classes: ClassVar[dict[str, type]] = {}
+    registered_user_preference_classes: ClassVar[dict[str, type]] = {}
 
     def __new__(cls, name, bases, attrs, *args, **kwargs):
         new_class = super().__new__(cls, name, bases, attrs, *args, **kwargs)
 
         if name != cls.__name__:
-            if name in cls._registered_user_preference_classes:
+            if name in cls.registered_user_preference_classes:
                 msg = (
                     f"Class named '{name}' was already defined at "
-                    f"{cls._registered_user_preference_classes[name]}."
+                    f"{cls.registered_user_preference_classes[name]}."
                     " Please choose a different class name!"
                 )
                 raise TypeError(msg)
-            cls._registered_user_preference_classes[name] = new_class
+            cls.registered_user_preference_classes[name] = new_class
 
         return new_class
 
@@ -63,7 +62,7 @@ def get_preference_class_from_name(
     ) -> type["_BaseUserPreferenceModel"]:
         preference_class: type[
             "_BaseUserPreferenceModel"
-        ] | None = cls._registered_user_preference_classes.get(preference_name, None)
+        ] | None = cls.registered_user_preference_classes.get(preference_name, None)
         if preference_class is None:
             raise NoPreferenceFoundError(preference_name)
         return preference_class
@@ -104,9 +103,6 @@ class UserServiceUserPreference(_BaseUserPreferenceModel):
     service_version: ServiceVersion = Field(
         ..., description="version of the service which manages the preference"
     )
-    file_path: Path = Field(
-        ..., description="path of the file where the preference is stored"
-    )
 
     def to_db(self) -> dict:
         return self.dict(exclude={"preference_type"})

packages/models-library/tests/test_service_settings_labels.py

@@ -42,7 +42,7 @@ class _Parametrization(NamedTuple):
     ),
     "dynamic-service": _Parametrization(
         example=SimcoreServiceLabels.Config.schema_extra["examples"][1],
-        items=4,
+        items=5,
         uses_dynamic_sidecar=True,
     ),
     "dynamic-service-with-compose-spec": _Parametrization(

packages/models-library/tests/test_user_preferences.py

@@ -82,7 +82,6 @@ def test_user_service_preferences(value: Any, mock_file_path: Path):
     )
     instance = parse_obj_as(UserServiceUserPreference, base_data)
     assert set(instance.to_db().keys()) == {
-        "file_path",
         "value",
         "service_key",
         "service_version",
@@ -93,7 +92,7 @@ def test_user_service_preferences(value: Any, mock_file_path: Path):
 def unregister_defined_classes() -> Iterator[None]:
     yield
     # pylint: disable=protected-access
-    _AutoRegisterMeta._registered_user_preference_classes.pop(  # noqa: SLF001
+    _AutoRegisterMeta.registered_user_preference_classes.pop(  # noqa: SLF001
         "Pref1", None
     )
 
@@ -120,7 +119,6 @@ def test__user_service__user_preference(
             "value": value,
             "service_key": service_key,
             "service_version": service_version,
-            "file_path": mock_file_path,
         }
     )
     assert isinstance(pref1, UserServiceUserPreference)

packages/service-integration/src/service_integration/osparc_config.py

@@ -215,6 +215,8 @@ class RuntimeConfig(BaseModel):
 
     callbacks_mapping: CallbacksMapping | None = Field(default_factory=dict)
     paths_mapping: PathMappingsLabel | None = None
+
+    user_preferences_path: Path | None = None
     boot_options: BootOptions = None
     min_visible_inputs: NonNegativeInt | None = None
 

packages/service-library/src/servicelib/file_utils.py

@@ -1,6 +1,7 @@
 import asyncio
 import hashlib
 import shutil
+import tempfile
 from pathlib import Path
 from typing import Final, Protocol
 
@@ -12,6 +13,12 @@
 
 CHUNK_4KB: Final[ByteSize] = parse_obj_as(ByteSize, "4kb")  # 4K blocks
 
+
+class AsyncStream(Protocol):
+    async def read(self, size: int = -1) -> bytes:
+        ...
+
+
 _shutil_rmtree = sync_to_async(shutil.rmtree)
 
 
@@ -33,9 +40,11 @@ async def remove_directory(
         await _shutil_rmtree(path, ignore_errors=ignore_errors)
 
 
-class AsyncStream(Protocol):
-    async def read(self, size: int = -1) -> bytes:
-        ...
+def get_temporary_path_name() -> Path:
+    """Only provides the temporary file name without creating the file on disk"""
+    # pylint: disable=W0212
+    tmp_path = Path(tempfile._get_default_tempdir())  # type: ignore #  noqa: SLF001
+    return tmp_path / next(tempfile._get_candidate_names())  # type: ignore # noqa: SLF001
 
 
 async def create_sha256_checksum(

packages/service-library/tests/test_file_utils.py

@@ -5,7 +5,7 @@
 
 import pytest
 from faker import Faker
-from servicelib.file_utils import remove_directory
+from servicelib.file_utils import get_temporary_path_name, remove_directory
 
 
 @pytest.fixture
@@ -80,3 +80,11 @@ async def test_remove_not_existing_directory_rasing_error(
         await remove_directory(
             path=missing_path, only_children=only_children, ignore_errors=False
         )
+
+
+def test_get_temporary_path_name_not_create_a_file_or_a_folder():
+    for _ in range(1000):
+        temp_path = get_temporary_path_name()
+        assert temp_path.exists() is False
+        assert temp_path.is_file() is False
+        assert temp_path.is_dir() is False

services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py

@@ -4,6 +4,7 @@
 from collections.abc import Mapping
 from enum import Enum
 from functools import cached_property
+from pathlib import Path
 from typing import Any, TypeAlias
 from uuid import UUID
 
@@ -372,6 +373,7 @@ def endpoint(self) -> AnyHttpUrl:
 
     paths_mapping: PathMappingsLabel  # overwrites in DynamicSidecarServiceLabels
 
+    user_preferences_path: Path | None = None
     callbacks_mapping: CallbacksMapping = Field(default_factory=dict)
 
     dynamic_sidecar_network_name: str = Field(
@@ -470,6 +472,7 @@ def from_http_request(
             "simcore_traefik_zone": names_helper.simcore_traefik_zone,
             "request_dns": request_dns,
             "request_scheme": request_scheme,
+            "user_preferences_path": simcore_service_labels.user_preferences_path,
             "proxy_service_name": names_helper.proxy_service_name,
             "request_simcore_user_agent": request_simcore_user_agent,
             "dynamic_sidecar": {"service_removal_state": {"can_save": can_save}},

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py

@@ -98,6 +98,10 @@ def _get_environment_variables(
         "SIMCORE_HOST_NAME": scheduler_data.service_name,
         "STORAGE_HOST": app_settings.DIRECTOR_V2_STORAGE.STORAGE_HOST,
         "STORAGE_PORT": f"{app_settings.DIRECTOR_V2_STORAGE.STORAGE_PORT}",
+        "DY_SIDECAR_SERVICE_KEY": scheduler_data.key,
+        "DY_SIDECAR_SERVICE_VERSION": scheduler_data.version,
+        "DY_SIDECAR_USER_PREFERENCES_PATH": f"{scheduler_data.user_preferences_path}",
+        "DY_SIDECAR_PRODUCT_NAME": f"{scheduler_data.product_name}",
     }
 
 
@@ -234,6 +238,19 @@ def get_dynamic_sidecar_spec(
             }
         )
 
+    if scheduler_data.user_preferences_path:
+        mounts.append(
+            DynamicSidecarVolumesPathsResolver.mount_user_preferences(
+                user_preferences_path=scheduler_data.user_preferences_path,
+                swarm_stack_name=dynamic_sidecar_settings.SWARM_STACK_NAME,
+                node_uuid=scheduler_data.node_uuid,
+                run_id=scheduler_data.run_id,
+                project_id=scheduler_data.project_id,
+                user_id=scheduler_data.user_id,
+                has_quota_support=has_quota_support,
+            )
+        )
+
     # PORTS -----------
     ports = []  # expose this service on an empty port
     if dynamic_sidecar_settings.DYNAMIC_SIDECAR_EXPOSE_PORT:

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/volumes.py

@@ -149,6 +149,7 @@ def mount_shared_store(
         project_id: ProjectID,
         user_id: UserID,
         swarm_stack_name: str,
+        *,
         has_quota_support: bool,
     ) -> dict[str, Any]:
         return cls.mount_entry(
@@ -161,6 +162,31 @@ def mount_shared_store(
             volume_size_limit="1M" if has_quota_support else None,
         )
 
+    @classmethod
+    def mount_user_preferences(
+        cls,
+        user_preferences_path: Path,
+        run_id: RunID,
+        node_uuid: NodeID,
+        project_id: ProjectID,
+        user_id: UserID,
+        swarm_stack_name: str,
+        *,
+        has_quota_support: bool,
+    ):
+        return cls.mount_entry(
+            swarm_stack_name=swarm_stack_name,
+            path=user_preferences_path,
+            node_uuid=node_uuid,
+            run_id=run_id,
+            project_id=project_id,
+            user_id=user_id,
+            # NOTE: the contents of this volume will be zipped and much
+            # be at most `_MAX_PREFERENCES_TOTAL_SIZE`, this 10M accounts
+            # for files and data that can be compressed a lot
+            volume_size_limit="10M" if has_quota_support else None,
+        )
+
     @classmethod
     def mount_r_clone(
         cls,

services/director-v2/tests/unit/test_modules_dynamic_sidecar_docker_service_specs_sidecar.py

@@ -18,11 +18,15 @@
     "DY_SIDECAR_NODE_ID",
     "DY_SIDECAR_PATH_INPUTS",
     "DY_SIDECAR_PATH_OUTPUTS",
+    "DY_SIDECAR_PRODUCT_NAME",
     "DY_SIDECAR_PROJECT_ID",
     "DY_SIDECAR_RUN_ID",
+    "DY_SIDECAR_SERVICE_KEY",
+    "DY_SIDECAR_SERVICE_VERSION",
     "DY_SIDECAR_STATE_EXCLUDE",
     "DY_SIDECAR_STATE_PATHS",
     "DY_SIDECAR_USER_ID",
+    "DY_SIDECAR_USER_PREFERENCES_PATH",
     "DY_SIDECAR_USER_SERVICES_HAVE_INTERNET_ACCESS",
     "DYNAMIC_SIDECAR_COMPOSE_NAMESPACE",
     "DYNAMIC_SIDECAR_LOG_LEVEL",

services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py

@@ -206,6 +206,10 @@ def expected_dynamic_sidecar_spec(
                         ': [{"service": "rt-web", "command": "ls", "timeout": 1.0}, {"service": "s4l-core", '
                         '"command": ["ls", "-lah"], "timeout": 1.0}]}'
                     ),
+                    "DY_SIDECAR_SERVICE_KEY": "simcore/services/dynamic/3dviewer",
+                    "DY_SIDECAR_SERVICE_VERSION": "2.4.5",
+                    "DY_SIDECAR_PRODUCT_NAME": osparc_product_name,
+                    "DY_SIDECAR_USER_PREFERENCES_PATH": "None",
                     "DY_SIDECAR_LOG_FORMAT_LOCAL_DEV_ENABLED": "True",
                     "POSTGRES_DB": "test",
                     "POSTGRES_HOST": "localhost",

services/dynamic-sidecar/requirements/_base.in

@@ -35,5 +35,6 @@ psutil
 pydantic
 python-magic # file type identification library. See 'magic.from_file(...)' NOTE: requires `libmagic`` installed
 PyYAML
+u-msgpack-python
 uvicorn
 watchdog

services/dynamic-sidecar/requirements/_base.txt

@@ -434,6 +434,8 @@ typing-extensions==4.7.1
     #   pydantic
     #   typer
     #   uvicorn
+u-msgpack-python==2.8.0
+    # via -r requirements/_base.in
 uvicorn==0.23.1
     # via
     #   -r requirements/../../../packages/service-library/requirements/_fastapi.in

services/dynamic-sidecar/src/simcore_service_dynamic_sidecar/core/application.py

@@ -23,6 +23,7 @@
 from ..modules.outputs import setup_outputs
 from ..modules.prometheus_metrics import setup_prometheus_metrics
 from ..modules.resource_tracking import setup_resource_tracking
+from ..modules.user_services_preferences import setup_user_services_preferences
 from .docker_compose_utils import docker_compose_down
 from .docker_logs import setup_background_log_fetcher
 from .error_handlers import http_error_handler, node_not_found_error_handler
@@ -162,6 +163,8 @@ def create_app():
 
     setup_attribute_monitor(app)
 
+    setup_user_services_preferences(app)
+
     if application_settings.are_prometheus_metrics_enabled:
         setup_prometheus_metrics(app)