✨Replace osparc variables in user services image labels (part 1) (#4805)

Co-authored-by: Andrei Neagu <[email protected]>

packages/models-library/src/models_library/osparc_variable_identifier.py

@@ -0,0 +1,106 @@
+from copy import deepcopy
+from typing import Any, TypeVar
+
+from pydantic import BaseModel, Field
+from pydantic.errors import PydanticErrorMixin
+
+from .utils.string_substitution import OSPARC_IDENTIFIER_PREFIX
+
+T = TypeVar("T")
+
+
+class OsparcVariableIdentifier(BaseModel):
+    # NOTE: To allow parametrized value, set the type to Union[OsparcVariableIdentifier, ...]
+    # NOTE: When dealing with str types, to avoid unexpected behavior, the following
+    # order is suggested `OsparcVariableIdentifier | str`
+    __root__: str = Field(
+        ..., regex=rf"^\${{?{OSPARC_IDENTIFIER_PREFIX}[A-Za-z0-9_]+}}?(:-.+)?$"
+    )
+
+    def _get_without_template_markers(self) -> str:
+        # $VAR
+        # ${VAR}
+        # ${VAR:-}
+        # ${VAR:-default}
+        # ${VAR:-{}}
+        if self.__root__.startswith("${"):
+            return self.__root__.removeprefix("${").removesuffix("}")
+        return self.__root__.removeprefix("$")
+
+    @property
+    def name(self) -> str:
+        return self._get_without_template_markers().split(":-")[0]
+
+    @property
+    def default_value(self) -> str | None:
+        parts = self._get_without_template_markers().split(":-")
+        return parts[1] if len(parts) > 1 else None
+
+
+class UnresolvedOsparcVariableIdentifierError(PydanticErrorMixin, TypeError):
+    msg_template = "Provided argument is unresolved: value={value}"
+
+
+def raise_if_unresolved(var: OsparcVariableIdentifier | T) -> T:
+    """Raise error or return original value
+
+    Use like below to make linters play nice.
+    ```
+    def example_func(par: OsparcVariableIdentifier | int) -> None:
+        _ = 12 + check_if_unresolved(par)
+    ```
+
+    Raises:
+        TypeError: if the the OsparcVariableIdentifier was unresolved
+    """
+    if isinstance(var, OsparcVariableIdentifier):
+        raise UnresolvedOsparcVariableIdentifierError(value=var)
+    return var
+
+
+def replace_osparc_variable_identifier(  # noqa: C901
+    obj: T, osparc_variables: dict[str, Any]
+) -> T:
+    """Replaces mostly in place an instance of `OsparcVariableIdentifier` with the
+    value provided inside `osparc_variables`.
+
+    NOTE: if the provided `obj` is instance of OsparcVariableIdentifier in place
+    replacement cannot be done. You need to assign it to the previous handler.
+
+    To be safe, always use like so:
+    ```
+    to_replace_obj = replace_osparc_variable_identifier(to_replace_obj)
+
+    Or like so:
+    ```
+    obj.to_replace_attribute =r eplace_osparc_variable_identifier(obj.to_replace_attribute)
+    ```
+    """
+
+    if isinstance(obj, OsparcVariableIdentifier):
+        if obj.name in osparc_variables:
+            return deepcopy(osparc_variables[obj.name])  # type: ignore
+        if obj.default_value is not None:
+            return deepcopy(obj.default_value)  # type: ignore
+    elif isinstance(obj, dict):
+        for key, value in obj.items():
+            obj[key] = replace_osparc_variable_identifier(value, osparc_variables)
+    elif isinstance(obj, BaseModel):
+        for key, value in obj.__dict__.items():
+            obj.__dict__[key] = replace_osparc_variable_identifier(
+                value, osparc_variables
+            )
+    if isinstance(obj, list):
+        for i, item in enumerate(obj):
+            obj[i] = replace_osparc_variable_identifier(item, osparc_variables)
+    elif isinstance(obj, tuple):
+        new_tuple = tuple(
+            replace_osparc_variable_identifier(item, osparc_variables) for item in obj
+        )
+        obj = new_tuple  # type: ignore
+    elif isinstance(obj, set):
+        new_set = {
+            replace_osparc_variable_identifier(item, osparc_variables) for item in obj
+        }
+        obj = new_set  # type: ignore
+    return obj

packages/models-library/src/models_library/service_settings_labels.py

@@ -1,7 +1,6 @@
 # pylint: disable=unsubscriptable-object
 
 import json
-import re
 from enum import Enum
 from functools import cached_property
 from pathlib import Path
@@ -10,7 +9,6 @@
 from pydantic import (
     BaseModel,
     ByteSize,
-    ConstrainedStr,
     Extra,
     Field,
     Json,
@@ -25,16 +23,10 @@
 from .generics import ListModel
 from .service_settings_nat_rule import NATRule
 from .services_resources import DEFAULT_SINGLE_SERVICE_NAME
-from .utils.string_substitution import OSPARC_IDENTIFIER_PREFIX
-
-# NOTE: To allow parametrized value, set the type to Union[OEnvSubstitutionStr, ...]
-
-
-class OEnvSubstitutionStr(ConstrainedStr):
-    regex = re.compile(rf"^\${OSPARC_IDENTIFIER_PREFIX}\w+$")
 
 
 class _BaseConfig:
+    arbitrary_types_allowed = True
     extra = Extra.forbid
     keep_untouched = (cached_property,)
 
@@ -98,6 +90,7 @@ def ensure_backwards_compatible_setting_type(cls, v):
         return v
 
     class Config(_BaseConfig):
+        allow_population_by_field_name = True
         schema_extra: ClassVar[dict[str, Any]] = {
             "examples": [
                 # constraints

packages/models-library/src/models_library/service_settings_nat_rule.py

@@ -4,6 +4,7 @@
 from pydantic import BaseModel, Extra, Field, parse_obj_as, validator
 
 from .basic_types import PortInt
+from .osparc_variable_identifier import OsparcVariableIdentifier, raise_if_unresolved
 
 # Cloudflare DNS server address
 DEFAULT_DNS_SERVER_ADDRESS: Final[str] = "1.1.1.1"  # NOSONAR
@@ -13,27 +14,40 @@
 class _PortRange(BaseModel):
     """`lower` and `upper` are included"""
 
-    lower: PortInt
-    upper: PortInt
+    lower: PortInt | OsparcVariableIdentifier
+    upper: PortInt | OsparcVariableIdentifier
 
     @validator("upper")
     @classmethod
     def lower_less_than_upper(cls, v, values) -> PortInt:
+        if isinstance(v, OsparcVariableIdentifier):
+            return v  # type: ignore # bypass validation if unresolved
+
         upper = v
-        lower: PortInt | None = values.get("lower")
+        lower: PortInt | OsparcVariableIdentifier | None = values.get("lower")
+
+        if lower and isinstance(lower, OsparcVariableIdentifier):
+            return v  # type: ignore # bypass validation if unresolved
+
         if lower is None or lower >= upper:
             msg = f"Condition not satisfied: lower={lower!r} < upper={upper!r}"
             raise ValueError(msg)
         return PortInt(v)
 
+    class Config:
+        arbitrary_types_allowed = True
+        validate_assignment = True
+
 
 class DNSResolver(BaseModel):
-    address: str = Field(
+    address: OsparcVariableIdentifier | str = Field(
         ..., description="this is not an url address is derived from IP address"
     )
-    port: PortInt
+    port: PortInt | OsparcVariableIdentifier
 
     class Config:
+        arbitrary_types_allowed = True
+        validate_assignment = True
         extra = Extra.allow
         schema_extra: ClassVar[dict[str, Any]] = {
             "examples": [
@@ -46,8 +60,8 @@ class Config:
 class NATRule(BaseModel):
     """Content of "simcore.service.containers-allowed-outgoing-permit-list" label"""
 
-    hostname: str
-    tcp_ports: list[_PortRange | PortInt]
+    hostname: OsparcVariableIdentifier | str
+    tcp_ports: list[PortInt | OsparcVariableIdentifier | _PortRange]
     dns_resolver: DNSResolver = Field(
         default_factory=lambda: DNSResolver(
             address=DEFAULT_DNS_SERVER_ADDRESS, port=DEFAULT_DNS_SERVER_PORT
@@ -58,6 +72,16 @@ class NATRule(BaseModel):
     def iter_tcp_ports(self) -> Generator[PortInt, None, None]:
         for port in self.tcp_ports:
             if isinstance(port, _PortRange):
-                yield from (PortInt(i) for i in range(port.lower, port.upper + 1))
+                yield from (
+                    PortInt(i)
+                    for i in range(
+                        raise_if_unresolved(port.lower),
+                        raise_if_unresolved(port.upper) + 1,
+                    )
+                )
             else:
-                yield port
+                yield raise_if_unresolved(port)
+
+    class Config:
+        arbitrary_types_allowed = True
+        validate_assignment = True

packages/models-library/src/models_library/utils/specs_substitution.py

@@ -1,6 +1,6 @@
+import json
 from typing import Any, TypeAlias, cast
 
-import yaml
 from pydantic import StrictBool, StrictFloat, StrictInt
 
 from .string_substitution import (
@@ -14,6 +14,14 @@
 SubstitutionValue: TypeAlias = StrictBool | StrictInt | StrictFloat | str
 
 
+def _json_dumps(data: dict[str, Any]) -> str:
+    return json.dumps(data)
+
+
+def _json_loads(str_data: str) -> dict[str, Any]:
+    return cast(dict[str, Any], json.loads(str_data))
+
+
 class SpecsSubstitutionsResolver:
     """
     Resolve specs dict by substituting identifiers
@@ -22,7 +30,7 @@ class SpecsSubstitutionsResolver:
 
     """
 
-    def __init__(self, specs: dict[str, Any], upgrade: bool):
+    def __init__(self, specs: dict[str, Any], *, upgrade: bool):
         self._template = self._create_text_template(specs, upgrade=upgrade)
         self._substitutions: SubstitutionsDict = SubstitutionsDict()
 
@@ -31,7 +39,7 @@ def _create_text_template(
         cls, specs: dict[str, Any], *, upgrade: bool
     ) -> TextTemplate:
         # convert to yaml (less symbols as in json)
-        service_spec_str: str = yaml.safe_dump(specs)
+        service_spec_str: str = _json_dumps(specs)
 
         if upgrade:  # legacy
             service_spec_str = substitute_all_legacy_identifiers(service_spec_str)
@@ -55,22 +63,33 @@ def substitutions(self):
         return self._substitutions
 
     def set_substitutions(
-        self, environs: dict[str, SubstitutionValue]
+        self, mappings: dict[str, SubstitutionValue]
     ) -> SubstitutionsDict:
-        """NOTE: ONLY targets identifiers declared in the specs"""
-        identifiers_needed = self.get_identifiers()
-
+        """
+        NOTE: ONLY targets identifiers declared in the specs
+        NOTE:`${identifier:-a_default_value}` will replace the identifier with `a_default_value`
+        if not provided
+        """
+
+        required_identifiers = self.get_identifiers()
+
+        required_identifiers_with_defaults: dict[str, str | None] = {}
+        for identifier in required_identifiers:
+            parts = identifier.split(":-")
+            required_identifiers_with_defaults[identifier] = (
+                parts[1] if ":-" in identifier else None
+            )
+
+        resolved_identifiers: dict[str, str] = {}
+        for identifier, default_value in required_identifiers_with_defaults.items():
+            if identifier in mappings:
+                resolved_identifiers[identifier] = cast(str, mappings[identifier])
+            elif default_value is not None:
+                resolved_identifiers[identifier] = default_value
         # picks only needed for substitution
-        self._substitutions = SubstitutionsDict(
-            {
-                identifier: environs[identifier]
-                for identifier in identifiers_needed
-                if identifier in environs
-            }
-        )
+        self._substitutions = SubstitutionsDict(resolved_identifiers)
         return self._substitutions
 
     def run(self) -> dict[str, Any]:
         new_specs_txt: str = self._template.safe_substitute(self._substitutions)
-        new_specs = yaml.safe_load(new_specs_txt)
-        return cast(dict[str, Any], new_specs)
+        return _json_loads(new_specs_txt)

packages/models-library/src/models_library/utils/string_substitution.py

@@ -52,9 +52,14 @@ class TextTemplate(Template):
     - `${identifier}` is equivalent to `$identifier`. It is required when valid identifier characters follow the
         placeholder but are not part of the placeholder, such as `"${noun}ification"`.
 
+    EXTENSION:
+    - `${identifier:-a_default_value}` is now also supported.
+
     SEE https://docs.python.org/3/library/string.html#template-strings
     """
 
+    idpattern = r"(?a:[_a-z][_a-z0-9]*)(?::-(.*?))?"
+
     if sys.version_info < (3, 11):
         # Backports methods added in py 3.11
         # NOTE: Keep it compatible with multiple version