✨ dynamic-sidecar saves and restores user services preferences

packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services_service.py

@@ -4,11 +4,10 @@
 
 from pydantic import BaseModel, Field
 
-from ..basic_regex import VERSION_RE
 from ..basic_types import PortInt
 from ..projects import ProjectID
 from ..projects_nodes_io import NodeID
-from ..services import DynamicServiceKey
+from ..services import DynamicServiceKey, ServiceVersion
 from ..services_enums import ServiceBootType, ServiceState
 from ..users import UserID
 
@@ -22,10 +21,9 @@ class CommonServiceDetails(BaseModel):
         ],
         alias="service_key",
     )
-    version: str = Field(
+    version: ServiceVersion = Field(
         ...,
         description="semantic version number of the node",
-        regex=VERSION_RE,
         examples=["1.0.0", "0.0.1"],
         alias="service_version",
     )

packages/models-library/src/models_library/service_settings_labels.py

@@ -210,10 +210,10 @@ class Config(_BaseConfig):
         schema_extra: ClassVar[dict[str, Any]] = {
             "examples": [
                 {
-                    "outputs_path": "/tmp/outputs",  # nosec
-                    "inputs_path": "/tmp/inputs",  # nosec
-                    "state_paths": ["/tmp/save_1", "/tmp_save_2"],  # nosec
-                    "state_exclude": ["/tmp/strip_me/*", "*.py"],  # nosec
+                    "outputs_path": "/tmp/outputs",  # noqa: S108 nosec
+                    "inputs_path": "/tmp/inputs",  # noqa: S108 nosec
+                    "state_paths": ["/tmp/save_1", "/tmp_save_2"],  # noqa: S108 nosec
+                    "state_exclude": ["/tmp/strip_me/*", "*.py"],  # noqa: S108 nosec
                 },
                 {
                     "outputs_path": "/t_out",
@@ -282,6 +282,15 @@ class DynamicSidecarServiceLabels(BaseModel):
         ),
     )
 
+    user_preferences_path: Path | None = Field(
+        None,
+        alias="simcore.service.user-preferences-path",
+        description=(
+            "path where the user user preferences folder "
+            "will be mounted in the user services"
+        ),
+    )
+
     restart_policy: RestartPolicy = Field(
         RestartPolicy.NO_RESTART,
         alias="simcore.service.restart-policy",
@@ -400,6 +409,30 @@ def ensure_callbacks_mapping_container_names_defined_in_compose_spec(
                     raise ValueError(err_msg)
         return v
 
+    @validator("user_preferences_path", pre=True)
+    @classmethod
+    def deserialize_from_json(cls, v):
+        return f"{v}".removeprefix('"').removesuffix('"')
+
+    @validator("user_preferences_path")
+    @classmethod
+    def user_preferences_path_no_included_in_other_volumes(
+        cls, v: CallbacksMapping, values
+    ):
+        paths_mapping: PathMappingsLabel | None = values.get("paths_mapping", None)
+        if paths_mapping is None:
+            return v
+
+        for test_path in [
+            paths_mapping.inputs_path,
+            paths_mapping.outputs_path,
+            *paths_mapping.state_paths,
+        ]:
+            if f"{test_path}".startswith(f"{v}"):
+                msg = f"user_preferences_path={v} cannot be a subpath of {test_path}"
+                raise ValueError(msg)
+        return v
+
     @root_validator
     @classmethod
     def not_allowed_in_both_specs(cls, values):
@@ -497,6 +530,9 @@ class Config(_BaseConfig):
                             }
                         }
                     ),
+                    "simcore.service.user-preferences-path": json.dumps(
+                        "/tmp/path_to_preferences"  # noqa: S108
+                    ),
                 },
                 # dynamic-service with compose spec
                 {

packages/models-library/src/models_library/user_preferences.py

@@ -1,5 +1,4 @@
 from enum import auto
-from pathlib import Path
 from typing import Annotated, Any, ClassVar, TypeAlias
 
 from pydantic import BaseModel, Field
@@ -13,20 +12,20 @@
 
 
 class _AutoRegisterMeta(ModelMetaclass):
-    _registered_user_preference_classes: ClassVar[dict[str, type]] = {}
+    registered_user_preference_classes: ClassVar[dict[str, type]] = {}
 
     def __new__(cls, name, bases, attrs, *args, **kwargs):
         new_class = super().__new__(cls, name, bases, attrs, *args, **kwargs)
 
         if name != cls.__name__:
-            if name in cls._registered_user_preference_classes:
+            if name in cls.registered_user_preference_classes:
                 msg = (
                     f"Class named '{name}' was already defined at "
-                    f"{cls._registered_user_preference_classes[name]}."
+                    f"{cls.registered_user_preference_classes[name]}."
                     " Please choose a different class name!"
                 )
                 raise TypeError(msg)
-            cls._registered_user_preference_classes[name] = new_class
+            cls.registered_user_preference_classes[name] = new_class
 
         return new_class
 
@@ -63,7 +62,7 @@ def get_preference_class_from_name(
     ) -> type["_BaseUserPreferenceModel"]:
         preference_class: type[
             "_BaseUserPreferenceModel"
-        ] | None = cls._registered_user_preference_classes.get(preference_name, None)
+        ] | None = cls.registered_user_preference_classes.get(preference_name, None)
         if preference_class is None:
             raise NoPreferenceFoundError(preference_name)
         return preference_class
@@ -104,9 +103,6 @@ class UserServiceUserPreference(_BaseUserPreferenceModel):
     service_version: ServiceVersion = Field(
         ..., description="version of the service which manages the preference"
     )
-    file_path: Path = Field(
-        ..., description="path of the file where the preference is stored"
-    )
 
     def to_db(self) -> dict:
         return self.dict(exclude={"preference_type"})

packages/models-library/tests/test_service_settings_labels.py

@@ -49,7 +49,7 @@ class _Parametrization(NamedTuple):
     ),
     "dynamic-service": _Parametrization(
         example=SimcoreServiceLabels.Config.schema_extra["examples"][1],
-        items=4,
+        items=5,
         uses_dynamic_sidecar=True,
     ),
     "dynamic-service-with-compose-spec": _Parametrization(
@@ -86,7 +86,10 @@ def test_service_settings():
     # ensure private attribute assignment
     for service_setting in simcore_settings_settings_label:
         # pylint: disable=protected-access
-        service_setting._destination_containers = ["random_value1", "random_value2"]
+        service_setting._destination_containers = [  # noqa: SLF001
+            "random_value1",
+            "random_value2",
+        ]
 
 
 @pytest.mark.parametrize("model_cls", [SimcoreServiceLabels])
@@ -108,7 +111,7 @@ def test_raises_error_if_http_entrypoint_is_missing():
     )
     del simcore_service_labels["simcore.service.container-http-entrypoint"]
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError):  # noqa: PT011
         SimcoreServiceLabels(**simcore_service_labels)
 
 
@@ -144,7 +147,7 @@ def test_raises_error_wrong_restart_policy():
     )
     simcore_service_labels["simcore.service.restart-policy"] = "__not_a_valid_policy__"
 
-    with pytest.raises(ValueError):
+    with pytest.raises(ValueError):  # noqa: PT011
         SimcoreServiceLabels(**simcore_service_labels)
 
 
@@ -406,10 +409,10 @@ def service_labels() -> dict[str, str]:
     return {
         "simcore.service.paths-mapping": json.dumps(
             {
-                "inputs_path": "/tmp/inputs",
-                "outputs_path": "/tmp/outputs",
-                "state_paths": ["/tmp/save_1", "/tmp_save_2"],
-                "state_exclude": ["/tmp/strip_me/*", "*.py"],
+                "inputs_path": "/tmp/inputs",  # noqa: S108
+                "outputs_path": "/tmp/outputs",  # noqa: S108
+                "state_paths": ["/tmp/save_1", "/tmp_save_2"],  # noqa: S108
+                "state_exclude": ["/tmp/strip_me/*", "*.py"],  # noqa: S108
             }
         ),
         "simcore.service.compose-spec": json.dumps(
@@ -426,7 +429,7 @@ def service_labels() -> dict[str, str]:
                         "runtime": "nvidia",
                         "init": True,
                         "environment": ["DISPLAY=${DISPLAY}"],
-                        "volumes": ["/tmp/.X11-unix:/tmp/.X11-unix"],
+                        "volumes": ["/tmp/.X11-unix:/tmp/.X11-unix"],  # noqa: S108
                     },
                 },
             }
@@ -483,8 +486,8 @@ def service_labels() -> dict[str, str]:
                     "value": [
                         {
                             "ReadOnly": True,
-                            "Source": "/tmp/.X11-unix",
-                            "Target": "/tmp/.X11-unix",
+                            "Source": "/tmp/.X11-unix",  # noqa: S108
+                            "Target": "/tmp/.X11-unix",  # noqa: S108
                             "Type": "bind",
                         }
                     ],
@@ -585,8 +588,27 @@ def test_resolving_some_service_labels_at_load_time(
 
     print(json.dumps(service_labels, indent=1))
 
+    # NOTE: that this model needs all values to be resolved before parsing them
+    # otherwise it might fail!! The question is whether these values can be resolved at this point
+    # NOTE: vendor values are in the database and therefore are available at this point
+    labels = SimcoreServiceLabels.parse_obj(service_labels)
+
+    print("After", labels.json(indent=1))
     formatted_json = service_meta.json(indent=1)
     print("After", formatted_json)
     for entry in vendor_environments:
         print(entry)
         assert entry not in formatted_json
+
+
+def test_user_preferences_path_is_part_of_exiting_volume():
+    labels_data = {
+        "simcore.service.paths-mapping": json.dumps(
+            PathMappingsLabel.Config.schema_extra["examples"][0]
+        ),
+        "simcore.service.user-preferences-path": json.dumps(
+            "/tmp/outputs"  # noqa: S108
+        ),
+    }
+    with pytest.raises(ValidationError, match="user_preferences_path=/tmp/outputs"):
+        assert DynamicSidecarServiceLabels.parse_raw(json.dumps(labels_data))

packages/models-library/tests/test_user_preferences.py

@@ -82,7 +82,6 @@ def test_user_service_preferences(value: Any, mock_file_path: Path):
     )
     instance = parse_obj_as(UserServiceUserPreference, base_data)
     assert set(instance.to_db().keys()) == {
-        "file_path",
         "value",
         "service_key",
         "service_version",
@@ -93,7 +92,7 @@ def test_user_service_preferences(value: Any, mock_file_path: Path):
 def unregister_defined_classes() -> Iterator[None]:
     yield
     # pylint: disable=protected-access
-    _AutoRegisterMeta._registered_user_preference_classes.pop(  # noqa: SLF001
+    _AutoRegisterMeta.registered_user_preference_classes.pop(  # noqa: SLF001
         "Pref1", None
     )
 
@@ -120,7 +119,6 @@ def test__user_service__user_preference(
             "value": value,
             "service_key": service_key,
             "service_version": service_version,
-            "file_path": mock_file_path,
         }
     )
     assert isinstance(pref1, UserServiceUserPreference)

packages/service-integration/src/service_integration/osparc_config.py

@@ -211,6 +211,8 @@ class RuntimeConfig(BaseModel):
 
     callbacks_mapping: CallbacksMapping | None = Field(default_factory=dict)
     paths_mapping: PathMappingsLabel | None = None
+
+    user_preferences_path: Path | None = None
     boot_options: BootOptions = None
     min_visible_inputs: NonNegativeInt | None = None
 

packages/service-library/src/servicelib/file_utils.py

@@ -12,6 +12,12 @@
 
 CHUNK_4KB: Final[ByteSize] = parse_obj_as(ByteSize, "4kb")  # 4K blocks
 
+
+class AsyncStream(Protocol):
+    async def read(self, size: int = -1) -> bytes:
+        ...
+
+
 _shutil_rmtree = sync_to_async(shutil.rmtree)
 
 
@@ -35,11 +41,6 @@ async def remove_directory(
         await _shutil_rmtree(path, ignore_errors=ignore_errors)
 
 
-class AsyncStream(Protocol):
-    async def read(self, size: int = -1) -> bytes:
-        ...
-
-
 async def create_sha256_checksum(
     async_stream: AsyncStream, *, chunk_size: ByteSize = CHUNK_4KB
 ) -> str:

services/director-v2/src/simcore_service_director_v2/models/dynamic_services_scheduler.py

@@ -4,6 +4,7 @@
 from collections.abc import Mapping
 from enum import Enum
 from functools import cached_property
+from pathlib import Path
 from typing import Any, TypeAlias
 from uuid import UUID
 
@@ -23,7 +24,15 @@
 from models_library.services import RunID
 from models_library.services_resources import ServiceResourcesDict
 from models_library.wallets import WalletInfo
-from pydantic import AnyHttpUrl, BaseModel, ConstrainedStr, Extra, Field, parse_obj_as
+from pydantic import (
+    AnyHttpUrl,
+    BaseModel,
+    ConstrainedStr,
+    Extra,
+    Field,
+    parse_obj_as,
+    validator,
+)
 from servicelib.error_codes import ErrorCodeStr
 from servicelib.exception_utils import DelayedExceptionHandler
 
@@ -372,6 +381,7 @@ def endpoint(self) -> AnyHttpUrl:
 
     paths_mapping: PathMappingsLabel  # overwrites in DynamicSidecarServiceLabels
 
+    user_preferences_path: Path | None = None
     callbacks_mapping: CallbacksMapping = Field(default_factory=dict)
 
     dynamic_sidecar_network_name: str = Field(
@@ -470,6 +480,7 @@ def from_http_request(
             "simcore_traefik_zone": names_helper.simcore_traefik_zone,
             "request_dns": request_dns,
             "request_scheme": request_scheme,
+            "user_preferences_path": simcore_service_labels.user_preferences_path,
             "proxy_service_name": names_helper.proxy_service_name,
             "request_simcore_user_agent": request_simcore_user_agent,
             "dynamic_sidecar": {"service_removal_state": {"can_save": can_save}},
@@ -479,6 +490,13 @@ def from_http_request(
             obj_dict["run_id"] = run_id
         return cls.parse_obj(obj_dict)  # type: ignore[no-any-return]
 
+    @validator("user_preferences_path", pre=True)
+    @classmethod
+    def strip_path_serialization_to_none(cls, v):
+        if v == "None":
+            return None
+        return v
+
     @classmethod
     def from_service_inspect(
         cls, service_inspect: Mapping[str, Any]

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_service_specs/sidecar.py

@@ -98,6 +98,10 @@ def _get_environment_variables(
         "SIMCORE_HOST_NAME": scheduler_data.service_name,
         "STORAGE_HOST": app_settings.DIRECTOR_V2_STORAGE.STORAGE_HOST,
         "STORAGE_PORT": f"{app_settings.DIRECTOR_V2_STORAGE.STORAGE_PORT}",
+        "DY_SIDECAR_SERVICE_KEY": scheduler_data.key,
+        "DY_SIDECAR_SERVICE_VERSION": scheduler_data.version,
+        "DY_SIDECAR_USER_PREFERENCES_PATH": f"{scheduler_data.user_preferences_path}",
+        "DY_SIDECAR_PRODUCT_NAME": f"{scheduler_data.product_name}",
     }
 
 
@@ -234,6 +238,19 @@ def get_dynamic_sidecar_spec(
             }
         )
 
+    if scheduler_data.user_preferences_path:
+        mounts.append(
+            DynamicSidecarVolumesPathsResolver.mount_user_preferences(
+                user_preferences_path=scheduler_data.user_preferences_path,
+                swarm_stack_name=dynamic_sidecar_settings.SWARM_STACK_NAME,
+                node_uuid=scheduler_data.node_uuid,
+                run_id=scheduler_data.run_id,
+                project_id=scheduler_data.project_id,
+                user_id=scheduler_data.user_id,
+                has_quota_support=has_quota_support,
+            )
+        )
+
     # PORTS -----------
     ports = []  # expose this service on an empty port
     if dynamic_sidecar_settings.DYNAMIC_SIDECAR_EXPOSE_PORT: