✨Computational backend: allow to set custom ec2 tags (⚠️ devops)

.pylintrc

@@ -92,7 +92,7 @@ persistent=yes
 py-version=3.10
 
 # Discover python modules and packages in the file system subtree.
-recursive=no
+recursive=true
 
 # When enabled, pylint would attempt to guess common misconfiguration and emit
 # user-friendly hints instead of false-positive error messages.

packages/aws-library/src/aws_library/ec2/client.py

@@ -7,7 +7,7 @@
 import botocore.exceptions
 from aiobotocore.session import ClientCreatorContext
 from aiocache import cached
-from pydantic import ByteSize
+from pydantic import ByteSize, parse_obj_as
 from servicelib.logging_utils import log_context
 from settings_library.ec2 import EC2Settings
 from types_aiobotocore_ec2 import EC2Client
@@ -170,7 +170,9 @@ async def start_aws_instance(
                     else None,
                     type=instance["InstanceType"],
                     state=instance["State"]["Name"],
-                    tags={tag["Key"]: tag["Value"] for tag in instance["Tags"]},
+                    tags=parse_obj_as(
+                        EC2Tags, {tag["Key"]: tag["Value"] for tag in instance["Tags"]}
+                    ),
                     resources=Resources(
                         cpus=instance_config.type.cpus, ram=instance_config.type.ram
                     ),
@@ -236,7 +238,10 @@ async def get_instances(
                             cpus=ec2_instance_types[0].cpus,
                             ram=ec2_instance_types[0].ram,
                         ),
-                        tags={tag["Key"]: tag["Value"] for tag in instance["Tags"]},
+                        tags=parse_obj_as(
+                            EC2Tags,
+                            {tag["Key"]: tag["Value"] for tag in instance["Tags"]},
+                        ),
                     )
                 )
         _logger.debug(

packages/aws-library/src/aws_library/ec2/models.py

@@ -1,4 +1,5 @@
 import datetime
+import re
 import tempfile
 from dataclasses import dataclass
 from typing import Any, ClassVar, TypeAlias
@@ -8,6 +9,7 @@
 from pydantic import (
     BaseModel,
     ByteSize,
+    ConstrainedStr,
     Extra,
     Field,
     NonNegativeFloat,
@@ -60,7 +62,20 @@ class EC2InstanceType:
 
 
 InstancePrivateDNSName: TypeAlias = str
-EC2Tags: TypeAlias = dict[str, str]
+
+
+class AWSTagKey(ConstrainedStr):
+    # see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions]
+    regex = re.compile(r"^(?!(_index|\.{1,2})$)[a-zA-Z0-9\+\-=\._:@]{1,128}$")
+
+
+class AWSTagValue(ConstrainedStr):
+    # see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions]
+    # quotes []{} were added as it allows to json encode. it seems to be accepted as a value
+    regex = re.compile(r"^[a-zA-Z0-9\s\+\-=\._:/@\"\'\[\]\{\}]{0,256}$")
+
+
+EC2Tags: TypeAlias = dict[AWSTagKey, AWSTagValue]
 
 
 @dataclass(frozen=True)

packages/aws-library/tests/test_ec2_models.py

@@ -4,8 +4,8 @@
 
 
 import pytest
-from aws_library.ec2.models import Resources
-from pydantic import ByteSize
+from aws_library.ec2.models import AWSTagKey, AWSTagValue, Resources
+from pydantic import ByteSize, ValidationError, parse_obj_as
 
 
 @pytest.mark.parametrize(
@@ -122,3 +122,13 @@ def test_resources_sub(a: Resources, b: Resources, result: Resources):
     assert a - b == result
     a -= b
     assert a == result
+
+
+@pytest.mark.parametrize("ec2_tag_key", ["", "/", " ", ".", "..", "_index"])
+def test_aws_tag_key_invalid(ec2_tag_key: str):
+    # for a key it raises
+    with pytest.raises(ValidationError):
+        parse_obj_as(AWSTagKey, ec2_tag_key)
+
+    # for a value it does not
+    parse_obj_as(AWSTagValue, ec2_tag_key)

services/autoscaling/src/simcore_service_autoscaling/core/settings.py

@@ -2,7 +2,7 @@
 from functools import cached_property
 from typing import Any, ClassVar, Final, cast
 
-from aws_library.ec2.models import EC2InstanceBootSpecific
+from aws_library.ec2.models import EC2InstanceBootSpecific, EC2Tags
 from fastapi import FastAPI
 from models_library.basic_types import (
     BootModeEnum,
@@ -101,6 +101,11 @@ class EC2InstancesSettings(BaseCustomSettings):
         description="Time after which an EC2 instance may be terminated (0<=T<=59 minutes, is automatically capped)"
         "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)",
     )
+    EC2_INSTANCES_CUSTOM_TAGS: EC2Tags = Field(
+        ...,
+        description="Allows to define tags that should be added to the created EC2 instance default tags. "
+        "a tag must have a key and an optional value. see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html]",
+    )
 
     @validator("EC2_INSTANCES_TIME_BEFORE_TERMINATION")
     @classmethod

services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py

@@ -518,7 +518,10 @@ async def _start_instances(
     ec2_client = get_ec2_client(app)
     app_settings = get_application_settings(app)
     assert app_settings.AUTOSCALING_EC2_INSTANCES  # nosec
-    new_instance_tags = auto_scaling_mode.get_ec2_tags(app)
+    new_instance_tags = (
+        auto_scaling_mode.get_ec2_tags(app)
+        | app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_CUSTOM_TAGS
+    )
     capped_needed_machines = {}
     try:
         capped_needed_machines = await _cap_needed_instances(

services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py

@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 
-from aws_library.ec2.models import EC2InstanceData, Resources
+from aws_library.ec2.models import EC2InstanceData, EC2Tags, Resources
 from fastapi import FastAPI
 from models_library.docker import DockerLabelKey
 from models_library.generated_models.docker_rest_api import Node as DockerNode
@@ -24,7 +24,7 @@ async def get_monitored_nodes(app: FastAPI) -> list[DockerNode]:
 
     @staticmethod
     @abstractmethod
-    def get_ec2_tags(app: FastAPI) -> dict[str, str]:
+    def get_ec2_tags(app: FastAPI) -> EC2Tags:
         ...
 
     @staticmethod

services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py

@@ -2,7 +2,7 @@
 import logging
 from collections.abc import Iterable
 
-from aws_library.ec2.models import EC2InstanceData, Resources
+from aws_library.ec2.models import EC2InstanceData, EC2Tags, Resources
 from fastapi import FastAPI
 from models_library.docker import (
     DOCKER_TASK_EC2_INSTANCE_TYPE_PLACEMENT_CONSTRAINT_KEY,
@@ -42,7 +42,7 @@ async def get_monitored_nodes(app: FastAPI) -> list[Node]:
         return await utils_docker.get_worker_nodes(get_docker_client(app))
 
     @staticmethod
-    def get_ec2_tags(app: FastAPI) -> dict[str, str]:
+    def get_ec2_tags(app: FastAPI) -> EC2Tags:
         app_settings = get_application_settings(app)
         return utils_ec2.get_ec2_tags_computational(app_settings)
 

services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py

@@ -1,6 +1,6 @@
 from collections.abc import Iterable
 
-from aws_library.ec2.models import EC2InstanceData, Resources
+from aws_library.ec2.models import EC2InstanceData, EC2Tags, Resources
 from fastapi import FastAPI
 from models_library.docker import DockerLabelKey
 from models_library.generated_models.docker_rest_api import Node, Task
@@ -31,7 +31,7 @@ async def get_monitored_nodes(app: FastAPI) -> list[Node]:
         )
 
     @staticmethod
-    def get_ec2_tags(app: FastAPI) -> dict[str, str]:
+    def get_ec2_tags(app: FastAPI) -> EC2Tags:
         app_settings = get_application_settings(app)
         return utils_ec2.get_ec2_tags_dynamic(app_settings)
 

services/autoscaling/src/simcore_service_autoscaling/utils/utils_ec2.py

@@ -8,7 +8,13 @@
 from collections.abc import Callable
 from textwrap import dedent
 
-from aws_library.ec2.models import EC2InstanceType, Resources
+from aws_library.ec2.models import (
+    AWSTagKey,
+    AWSTagValue,
+    EC2InstanceType,
+    EC2Tags,
+    Resources,
+)
 
 from .._meta import VERSION
 from ..core.errors import ConfigurationError, Ec2InstanceNotFoundError
@@ -17,32 +23,40 @@
 logger = logging.getLogger(__name__)
 
 
-def get_ec2_tags_dynamic(app_settings: ApplicationSettings) -> dict[str, str]:
+def get_ec2_tags_dynamic(app_settings: ApplicationSettings) -> EC2Tags:
     assert app_settings.AUTOSCALING_NODES_MONITORING  # nosec
     assert app_settings.AUTOSCALING_EC2_INSTANCES  # nosec
     return {
-        "io.simcore.autoscaling.version": f"{VERSION}",
-        "io.simcore.autoscaling.monitored_nodes_labels": json.dumps(
-            app_settings.AUTOSCALING_NODES_MONITORING.NODES_MONITORING_NODE_LABELS
+        AWSTagKey("io.simcore.autoscaling.version"): AWSTagValue(f"{VERSION}"),
+        AWSTagKey("io.simcore.autoscaling.monitored_nodes_labels"): AWSTagValue(
+            json.dumps(
+                app_settings.AUTOSCALING_NODES_MONITORING.NODES_MONITORING_NODE_LABELS
+            )
         ),
-        "io.simcore.autoscaling.monitored_services_labels": json.dumps(
-            app_settings.AUTOSCALING_NODES_MONITORING.NODES_MONITORING_SERVICE_LABELS
+        AWSTagKey("io.simcore.autoscaling.monitored_services_labels"): AWSTagValue(
+            json.dumps(
+                app_settings.AUTOSCALING_NODES_MONITORING.NODES_MONITORING_SERVICE_LABELS
+            )
         ),
         # NOTE: this one gets special treatment in AWS GUI and is applied to the name of the instance
-        "Name": f"{app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_NAME_PREFIX}-{app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_KEY_NAME}",
+        AWSTagKey("Name"): AWSTagValue(
+            f"{app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_NAME_PREFIX}-{app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_KEY_NAME}"
+        ),
     }
 
 
-def get_ec2_tags_computational(app_settings: ApplicationSettings) -> dict[str, str]:
+def get_ec2_tags_computational(app_settings: ApplicationSettings) -> EC2Tags:
     assert app_settings.AUTOSCALING_DASK  # nosec
     assert app_settings.AUTOSCALING_EC2_INSTANCES  # nosec
     return {
-        "io.simcore.autoscaling.version": f"{VERSION}",
-        "io.simcore.autoscaling.dask-scheduler_url": json.dumps(
-            app_settings.AUTOSCALING_DASK.DASK_MONITORING_URL
+        AWSTagKey("io.simcore.autoscaling.version"): AWSTagValue(f"{VERSION}"),
+        AWSTagKey("io.simcore.autoscaling.dask-scheduler_url"): AWSTagValue(
+            f"{app_settings.AUTOSCALING_DASK.DASK_MONITORING_URL}"
         ),
         # NOTE: this one gets special treatment in AWS GUI and is applied to the name of the instance
-        "Name": f"{app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_NAME_PREFIX}-{app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_KEY_NAME}",
+        AWSTagKey("Name"): AWSTagValue(
+            f"{app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_NAME_PREFIX}-{app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_KEY_NAME}"
+        ),
     }
 
 

services/autoscaling/tests/unit/conftest.py

@@ -132,6 +132,13 @@ def app_environment(
                     for ec2_type_name in ec2_instances
                 }
             ),
+            "EC2_INSTANCES_CUSTOM_TAGS": json.dumps(
+                {
+                    "user_id": "32",
+                    "wallet_id": "3245",
+                    "osparc-tag": "some whatever value",
+                }
+            ),
         },
     )
     return mock_env_devel_environment | envs

services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py

@@ -115,6 +115,9 @@ async def _assert_ec2_instances(
                 "io.simcore.autoscaling.version",
                 "io.simcore.autoscaling.dask-scheduler_url",
                 "Name",
+                "user_id",
+                "wallet_id",
+                "osparc-tag",
             ]
             for tag_dict in instance["Tags"]:
                 assert "Key" in tag_dict

services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py

@@ -381,6 +381,9 @@ async def _assert_ec2_instances(
                 "io.simcore.autoscaling.monitored_nodes_labels",
                 "io.simcore.autoscaling.monitored_services_labels",
                 "Name",
+                "user_id",
+                "wallet_id",
+                "osparc-tag",
             ]
             for tag_dict in instance["Tags"]:
                 assert "Key" in tag_dict

services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py

@@ -2,7 +2,7 @@
 from functools import cached_property
 from typing import Any, ClassVar, Final, cast
 
-from aws_library.ec2.models import EC2InstanceBootSpecific
+from aws_library.ec2.models import EC2InstanceBootSpecific, EC2Tags
 from fastapi import FastAPI
 from models_library.basic_types import (
     BootModeEnum,
@@ -85,6 +85,12 @@ class WorkersEC2InstancesSettings(BaseCustomSettings):
         "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)",
     )
 
+    WORKERS_EC2_INSTANCES_CUSTOM_TAGS: EC2Tags = Field(
+        ...,
+        description="Allows to define tags that should be added to the created EC2 instance default tags. "
+        "a tag must have a key and an optional value. see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html]",
+    )
+
     @validator("WORKERS_EC2_INSTANCES_ALLOWED_TYPES")
     @classmethod
     def check_valid_instance_names(
@@ -129,6 +135,11 @@ class PrimaryEC2InstancesSettings(BaseCustomSettings):
         " (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html),"
         "this is required to start a new EC2 instance",
     )
+    PRIMARY_EC2_INSTANCES_CUSTOM_TAGS: EC2Tags = Field(
+        ...,
+        description="Allows to define tags that should be added to the created EC2 instance default tags. "
+        "a tag must have a key and an optional value. see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html]",
+    )
 
     @validator("PRIMARY_EC2_INSTANCES_ALLOWED_TYPES")
     @classmethod

services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml

@@ -61,6 +61,7 @@ services:
       EC2_INSTANCES_SECURITY_GROUP_IDS: ${WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS}
       EC2_INSTANCES_SUBNET_ID: ${WORKERS_EC2_INSTANCES_SUBNET_ID}
       EC2_INSTANCES_TIME_BEFORE_TERMINATION: ${WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION}
+      EC2_INSTANCES_CUSTOM_TAGS: ${WORKERS_EC2_INSTANCES_CUSTOM_TAGS}
       LOG_FORMAT_LOCAL_DEV_ENABLED: 1
       LOG_LEVEL: ${LOG_LEVEL:-WARNING}
       REDIS_HOST: redis

services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters.py

@@ -3,6 +3,8 @@
 
 from aws_library.ec2.client import SimcoreEC2API
 from aws_library.ec2.models import (
+    AWSTagKey,
+    AWSTagValue,
     EC2InstanceBootSpecific,
     EC2InstanceConfig,
     EC2InstanceData,
@@ -69,10 +71,14 @@ async def create_cluster(
         tags=creation_ec2_tags(app_settings, user_id=user_id, wallet_id=wallet_id),
         startup_script=create_startup_script(
             app_settings,
-            get_cluster_name(
+            cluster_machines_name_prefix=get_cluster_name(
                 app_settings, user_id=user_id, wallet_id=wallet_id, is_manager=False
             ),
-            ec2_instance_boot_specs,
+            ec2_boot_specific=ec2_instance_boot_specs,
+            additional_custom_tags={
+                AWSTagKey("user_id"): AWSTagValue(f"{user_id}"),
+                AWSTagKey("wallet_id"): AWSTagValue(f"{wallet_id}"),
+            },
         ),
         ami_id=ec2_instance_boot_specs.ami_id,
         key_name=app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_KEY_NAME,

services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py

@@ -4,7 +4,7 @@
 import json
 from typing import Any, Final
 
-from aws_library.ec2.models import EC2InstanceBootSpecific, EC2InstanceData
+from aws_library.ec2.models import EC2InstanceBootSpecific, EC2InstanceData, EC2Tags
 from fastapi.encoders import jsonable_encoder
 from models_library.api_schemas_clusters_keeper.clusters import (
     ClusterState,
@@ -32,8 +32,10 @@ def _docker_compose_yml_base64_encoded() -> str:
 
 def create_startup_script(
     app_settings: ApplicationSettings,
+    *,
     cluster_machines_name_prefix: str,
     ec2_boot_specific: EC2InstanceBootSpecific,
+    additional_custom_tags: EC2Tags,
 ) -> str:
     assert app_settings.CLUSTERS_KEEPER_EC2_ACCESS  # nosec
     assert app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES  # nosec
@@ -58,6 +60,7 @@ def _convert_to_env_dict(entries: dict[str, Any]) -> str:
         f"WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS={_convert_to_env_list(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS)}",
         f"WORKERS_EC2_INSTANCES_SUBNET_ID={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SUBNET_ID}",
         f"WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION}",
+        f"WORKERS_EC2_INSTANCES_CUSTOM_TAGS={_convert_to_env_dict(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_CUSTOM_TAGS | additional_custom_tags)}",
         f"LOG_LEVEL={app_settings.LOG_LEVEL}",
     ]